[mirror_ubuntu-jammy-kernel.git] / drivers / crypto / cavium / cpt / cptvf_reqmanager.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2016 Cavium, Inc.
 */

#include "cptvf.h"
#include "cptvf_algs.h"
#include "request_manager.h"

/**
 * get_free_pending_entry - get free entry from pending queue
 * @param pqinfo: pending_qinfo structure
 * @param qno: queue number
 */
static struct pending_entry *get_free_pending_entry(struct pending_queue *q,
						    int qlen)
{
	struct pending_entry *ent = NULL;

	ent = &q->head[q->rear];
	if (unlikely(ent->busy)) {
		ent = NULL;
		goto no_free_entry;
	}

	q->rear++;
	if (unlikely(q->rear == qlen))
		q->rear = 0;

no_free_entry:
	return ent;
}

static inline void pending_queue_inc_front(struct pending_qinfo *pqinfo,
					   int qno)
{
	struct pending_queue *queue = &pqinfo->queue[qno];

	queue->front++;
	if (unlikely(queue->front == pqinfo->qlen))
		queue->front = 0;
}

static int setup_sgio_components(struct cpt_vf *cptvf, struct buf_ptr *list,
				 int buf_count, u8 *buffer)
{
	int ret = 0, i, j;
	int components;
	struct sglist_component *sg_ptr = NULL;
	struct pci_dev *pdev = cptvf->pdev;

	if (unlikely(!list)) {
		dev_err(&pdev->dev, "Input List pointer is NULL\n");
		return -EFAULT;
	}

	for (i = 0; i < buf_count; i++) {
		if (likely(list[i].vptr)) {
			list[i].dma_addr = dma_map_single(&pdev->dev,
							  list[i].vptr,
							  list[i].size,
							  DMA_BIDIRECTIONAL);
			if (unlikely(dma_mapping_error(&pdev->dev,
						       list[i].dma_addr))) {
				dev_err(&pdev->dev, "DMA map kernel buffer failed for component: %d\n",
					i);
				ret = -EIO;
				goto sg_cleanup;
			}
		}
	}

	components = buf_count / 4;
	sg_ptr = (struct sglist_component *)buffer;
	for (i = 0; i < components; i++) {
		sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
		sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
		sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
		sg_ptr->u.s.len3 = cpu_to_be16(list[i * 4 + 3].size);
		sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
		sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
		sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
		sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr);
		sg_ptr++;
	}

	components = buf_count % 4;

	switch (components) {
	case 3:
		sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
		sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
		fallthrough;
	case 2:
		sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
		sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
		fallthrough;
	case 1:
		sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
		sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
		break;
	default:
		break;
	}

	return ret;

sg_cleanup:
	for (j = 0; j < i; j++) {
		if (list[j].dma_addr) {
			dma_unmap_single(&pdev->dev, list[i].dma_addr,
					 list[i].size, DMA_BIDIRECTIONAL);
		}

		list[j].dma_addr = 0;
	}

	return ret;
}

static inline int setup_sgio_list(struct cpt_vf *cptvf,
				  struct cpt_info_buffer *info,
				  struct cpt_request_info *req)
{
	u16 g_sz_bytes = 0, s_sz_bytes = 0;
	int ret = 0;
	struct pci_dev *pdev = cptvf->pdev;

	if (req->incnt > MAX_SG_IN_CNT || req->outcnt > MAX_SG_OUT_CNT) {
		dev_err(&pdev->dev, "Request SG components are higher than supported\n");
		ret = -EINVAL;
		goto  scatter_gather_clean;
	}

	/* Setup gather (input) components */
	g_sz_bytes = ((req->incnt + 3) / 4) * sizeof(struct sglist_component);
	info->gather_components = kzalloc(g_sz_bytes, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
	if (!info->gather_components) {
		ret = -ENOMEM;
		goto  scatter_gather_clean;
	}

	ret = setup_sgio_components(cptvf, req->in,
				    req->incnt,
				    info->gather_components);
	if (ret) {
		dev_err(&pdev->dev, "Failed to setup gather list\n");
		ret = -EFAULT;
		goto  scatter_gather_clean;
	}

	/* Setup scatter (output) components */
	s_sz_bytes = ((req->outcnt + 3) / 4) * sizeof(struct sglist_component);
	info->scatter_components = kzalloc(s_sz_bytes, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
	if (!info->scatter_components) {
		ret = -ENOMEM;
		goto  scatter_gather_clean;
	}

	ret = setup_sgio_components(cptvf, req->out,
				    req->outcnt,
				    info->scatter_components);
	if (ret) {
		dev_err(&pdev->dev, "Failed to setup gather list\n");
		ret = -EFAULT;
		goto  scatter_gather_clean;
	}

	/* Create and initialize DPTR */
	info->dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE;
	info->in_buffer = kzalloc(info->dlen, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
	if (!info->in_buffer) {
		ret = -ENOMEM;
		goto  scatter_gather_clean;
	}

	((__be16 *)info->in_buffer)[0] = cpu_to_be16(req->outcnt);
	((__be16 *)info->in_buffer)[1] = cpu_to_be16(req->incnt);
	((__be16 *)info->in_buffer)[2] = 0;
	((__be16 *)info->in_buffer)[3] = 0;

	memcpy(&info->in_buffer[8], info->gather_components,
	       g_sz_bytes);
	memcpy(&info->in_buffer[8 + g_sz_bytes],
	       info->scatter_components, s_sz_bytes);

	info->dptr_baddr = dma_map_single(&pdev->dev,
					  (void *)info->in_buffer,
					  info->dlen,
					  DMA_BIDIRECTIONAL);
	if (dma_mapping_error(&pdev->dev, info->dptr_baddr)) {
		dev_err(&pdev->dev, "Mapping DPTR Failed %d\n", info->dlen);
		ret = -EIO;
		goto  scatter_gather_clean;
	}

	/* Create and initialize RPTR */
	info->out_buffer = kzalloc(COMPLETION_CODE_SIZE, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
	if (!info->out_buffer) {
		ret = -ENOMEM;
		goto scatter_gather_clean;
	}

	*((u64 *)info->out_buffer) = ~((u64)COMPLETION_CODE_INIT);
	info->alternate_caddr = (u64 *)info->out_buffer;
	info->rptr_baddr = dma_map_single(&pdev->dev,
					  (void *)info->out_buffer,
					  COMPLETION_CODE_SIZE,
					  DMA_BIDIRECTIONAL);
	if (dma_mapping_error(&pdev->dev, info->rptr_baddr)) {
		dev_err(&pdev->dev, "Mapping RPTR Failed %d\n",
			COMPLETION_CODE_SIZE);
		ret = -EIO;
		goto  scatter_gather_clean;
	}

	return 0;

scatter_gather_clean:
	return ret;
}

static int send_cpt_command(struct cpt_vf *cptvf, union cpt_inst_s *cmd,
		     u32 qno)
{
	struct pci_dev *pdev = cptvf->pdev;
	struct command_qinfo *qinfo = NULL;
	struct command_queue *queue;
	struct command_chunk *chunk;
	u8 *ent;
	int ret = 0;

	if (unlikely(qno >= cptvf->nr_queues)) {
		dev_err(&pdev->dev, "Invalid queue (qno: %d, nr_queues: %d)\n",
			qno, cptvf->nr_queues);
		return -EINVAL;
	}

	qinfo = &cptvf->cqinfo;
	queue = &qinfo->queue[qno];
	/* lock commad queue */
	spin_lock(&queue->lock);
	ent = &queue->qhead->head[queue->idx * qinfo->cmd_size];
	memcpy(ent, (void *)cmd, qinfo->cmd_size);

	if (++queue->idx >= queue->qhead->size / 64) {
		struct hlist_node *node;

		hlist_for_each(node, &queue->chead) {
			chunk = hlist_entry(node, struct command_chunk,
					    nextchunk);
			if (chunk == queue->qhead) {
				continue;
			} else {
				queue->qhead = chunk;
				break;
			}
		}
		queue->idx = 0;
	}
	/* make sure all memory stores are done before ringing doorbell */
	smp_wmb();
	cptvf_write_vq_doorbell(cptvf, 1);
	/* unlock command queue */
	spin_unlock(&queue->lock);

	return ret;
}

static void do_request_cleanup(struct cpt_vf *cptvf,
			struct cpt_info_buffer *info)
{
	int i;
	struct pci_dev *pdev = cptvf->pdev;
	struct cpt_request_info *req;

	if (info->dptr_baddr)
		dma_unmap_single(&pdev->dev, info->dptr_baddr,
				 info->dlen, DMA_BIDIRECTIONAL);

	if (info->rptr_baddr)
		dma_unmap_single(&pdev->dev, info->rptr_baddr,
				 COMPLETION_CODE_SIZE, DMA_BIDIRECTIONAL);

	if (info->comp_baddr)
		dma_unmap_single(&pdev->dev, info->comp_baddr,
				 sizeof(union cpt_res_s), DMA_BIDIRECTIONAL);

	if (info->req) {
		req = info->req;
		for (i = 0; i < req->outcnt; i++) {
			if (req->out[i].dma_addr)
				dma_unmap_single(&pdev->dev,
						 req->out[i].dma_addr,
						 req->out[i].size,
						 DMA_BIDIRECTIONAL);
		}

		for (i = 0; i < req->incnt; i++) {
			if (req->in[i].dma_addr)
				dma_unmap_single(&pdev->dev,
						 req->in[i].dma_addr,
						 req->in[i].size,
						 DMA_BIDIRECTIONAL);
		}
	}

	kfree_sensitive(info->scatter_components);
	kfree_sensitive(info->gather_components);
	kfree_sensitive(info->out_buffer);
	kfree_sensitive(info->in_buffer);
	kfree_sensitive((void *)info->completion_addr);
	kfree_sensitive(info);
}

static void do_post_process(struct cpt_vf *cptvf, struct cpt_info_buffer *info)
{
	struct pci_dev *pdev = cptvf->pdev;

	if (!info) {
		dev_err(&pdev->dev, "incorrect cpt_info_buffer for post processing\n");
		return;
	}

	do_request_cleanup(cptvf, info);
}

static inline void process_pending_queue(struct cpt_vf *cptvf,
					 struct pending_qinfo *pqinfo,
					 int qno)
{
	struct pci_dev *pdev = cptvf->pdev;
	struct pending_queue *pqueue = &pqinfo->queue[qno];
	struct pending_entry *pentry = NULL;
	struct cpt_info_buffer *info = NULL;
	union cpt_res_s *status = NULL;
	unsigned char ccode;

	while (1) {
		spin_lock_bh(&pqueue->lock);
		pentry = &pqueue->head[pqueue->front];
		if (unlikely(!pentry->busy)) {
			spin_unlock_bh(&pqueue->lock);
			break;
		}

		info = (struct cpt_info_buffer *)pentry->post_arg;
		if (unlikely(!info)) {
			dev_err(&pdev->dev, "Pending Entry post arg NULL\n");
			pending_queue_inc_front(pqinfo, qno);
			spin_unlock_bh(&pqueue->lock);
			continue;
		}

		status = (union cpt_res_s *)pentry->completion_addr;
		ccode = status->s.compcode;
		if ((status->s.compcode == CPT_COMP_E_FAULT) ||
		    (status->s.compcode == CPT_COMP_E_SWERR)) {
			dev_err(&pdev->dev, "Request failed with %s\n",
				(status->s.compcode == CPT_COMP_E_FAULT) ?
				"DMA Fault" : "Software error");
			pentry->completion_addr = NULL;
			pentry->busy = false;
			atomic64_dec((&pqueue->pending_count));
			pentry->post_arg = NULL;
			pending_queue_inc_front(pqinfo, qno);
			do_request_cleanup(cptvf, info);
			spin_unlock_bh(&pqueue->lock);
			break;
		} else if (status->s.compcode == COMPLETION_CODE_INIT) {
			/* check for timeout */
			if (time_after_eq(jiffies,
					  (info->time_in +
					  (CPT_COMMAND_TIMEOUT * HZ)))) {
				dev_err(&pdev->dev, "Request timed out");
				pentry->completion_addr = NULL;
				pentry->busy = false;
				atomic64_dec((&pqueue->pending_count));
				pentry->post_arg = NULL;
				pending_queue_inc_front(pqinfo, qno);
				do_request_cleanup(cptvf, info);
				spin_unlock_bh(&pqueue->lock);
				break;
			} else if ((*info->alternate_caddr ==
				(~COMPLETION_CODE_INIT)) &&
				(info->extra_time < TIME_IN_RESET_COUNT)) {
				info->time_in = jiffies;
				info->extra_time++;
				spin_unlock_bh(&pqueue->lock);
				break;
			}
		}

		pentry->completion_addr = NULL;
		pentry->busy = false;
		pentry->post_arg = NULL;
		atomic64_dec((&pqueue->pending_count));
		pending_queue_inc_front(pqinfo, qno);
		spin_unlock_bh(&pqueue->lock);

		do_post_process(info->cptvf, info);
		/*
		 * Calling callback after we find
		 * that the request has been serviced
		 */
		pentry->callback(ccode, pentry->callback_arg);
	}
}

int process_request(struct cpt_vf *cptvf, struct cpt_request_info *req)
{
	int ret = 0, clear = 0, queue = 0;
	struct cpt_info_buffer *info = NULL;
	struct cptvf_request *cpt_req = NULL;
	union ctrl_info *ctrl = NULL;
	union cpt_res_s *result = NULL;
	struct pending_entry *pentry = NULL;
	struct pending_queue *pqueue = NULL;
	struct pci_dev *pdev = cptvf->pdev;
	u8 group = 0;
	struct cpt_vq_command vq_cmd;
	union cpt_inst_s cptinst;

	info = kzalloc(sizeof(*info), req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
	if (unlikely(!info)) {
		dev_err(&pdev->dev, "Unable to allocate memory for info_buffer\n");
		return -ENOMEM;
	}

	cpt_req = (struct cptvf_request *)&req->req;
	ctrl = (union ctrl_info *)&req->ctrl;

	info->cptvf = cptvf;
	group = ctrl->s.grp;
	ret = setup_sgio_list(cptvf, info, req);
	if (ret) {
		dev_err(&pdev->dev, "Setting up SG list failed");
		goto request_cleanup;
	}

	cpt_req->dlen = info->dlen;
	/*
	 * Get buffer for union cpt_res_s response
	 * structure and its physical address
	 */
	info->completion_addr = kzalloc(sizeof(union cpt_res_s), req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
	if (unlikely(!info->completion_addr)) {
		dev_err(&pdev->dev, "Unable to allocate memory for completion_addr\n");
		ret = -ENOMEM;
		goto request_cleanup;
	}

	result = (union cpt_res_s *)info->completion_addr;
	result->s.compcode = COMPLETION_CODE_INIT;
	info->comp_baddr = dma_map_single(&pdev->dev,
					       (void *)info->completion_addr,
					       sizeof(union cpt_res_s),
					       DMA_BIDIRECTIONAL);
	if (dma_mapping_error(&pdev->dev, info->comp_baddr)) {
		dev_err(&pdev->dev, "mapping compptr Failed %lu\n",
			sizeof(union cpt_res_s));
		ret = -EFAULT;
		goto  request_cleanup;
	}

	/* Fill the VQ command */
	vq_cmd.cmd.u64 = 0;
	vq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags);
	vq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1);
	vq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2);
	vq_cmd.cmd.s.dlen   = cpu_to_be16(cpt_req->dlen);

	vq_cmd.dptr = info->dptr_baddr;
	vq_cmd.rptr = info->rptr_baddr;
	vq_cmd.cptr.u64 = 0;
	vq_cmd.cptr.s.grp = group;
	/* Get Pending Entry to submit command */
	/* Always queue 0, because 1 queue per VF */
	queue = 0;
	pqueue = &cptvf->pqinfo.queue[queue];

	if (atomic64_read(&pqueue->pending_count) > PENDING_THOLD) {
		dev_err(&pdev->dev, "pending threshold reached\n");
		process_pending_queue(cptvf, &cptvf->pqinfo, queue);
	}

get_pending_entry:
	spin_lock_bh(&pqueue->lock);
	pentry = get_free_pending_entry(pqueue, cptvf->pqinfo.qlen);
	if (unlikely(!pentry)) {
		spin_unlock_bh(&pqueue->lock);
		if (clear == 0) {
			process_pending_queue(cptvf, &cptvf->pqinfo, queue);
			clear = 1;
			goto get_pending_entry;
		}
		dev_err(&pdev->dev, "Get free entry failed\n");
		dev_err(&pdev->dev, "queue: %d, rear: %d, front: %d\n",
			queue, pqueue->rear, pqueue->front);
		ret = -EFAULT;
		goto request_cleanup;
	}

	pentry->completion_addr = info->completion_addr;
	pentry->post_arg = (void *)info;
	pentry->callback = req->callback;
	pentry->callback_arg = req->callback_arg;
	info->pentry = pentry;
	pentry->busy = true;
	atomic64_inc(&pqueue->pending_count);

	/* Send CPT command */
	info->pentry = pentry;
	info->time_in = jiffies;
	info->req = req;

	/* Create the CPT_INST_S type command for HW intrepretation */
	cptinst.s.doneint = true;
	cptinst.s.res_addr = (u64)info->comp_baddr;
	cptinst.s.tag = 0;
	cptinst.s.grp = 0;
	cptinst.s.wq_ptr = 0;
	cptinst.s.ei0 = vq_cmd.cmd.u64;
	cptinst.s.ei1 = vq_cmd.dptr;
	cptinst.s.ei2 = vq_cmd.rptr;
	cptinst.s.ei3 = vq_cmd.cptr.u64;

	ret = send_cpt_command(cptvf, &cptinst, queue);
	spin_unlock_bh(&pqueue->lock);
	if (unlikely(ret)) {
		dev_err(&pdev->dev, "Send command failed for AE\n");
		ret = -EFAULT;
		goto request_cleanup;
	}

	return 0;

request_cleanup:
	dev_dbg(&pdev->dev, "Failed to submit CPT command\n");
	do_request_cleanup(cptvf, info);

	return ret;
}

void vq_post_process(struct cpt_vf *cptvf, u32 qno)
{
	struct pci_dev *pdev = cptvf->pdev;

	if (unlikely(qno > cptvf->nr_queues)) {
		dev_err(&pdev->dev, "Request for post processing on invalid pending queue: %u\n",
			qno);
		return;
	}

	process_pending_queue(cptvf, &cptvf->pqinfo, qno);
}

int cptvf_do_request(void *vfdev, struct cpt_request_info *req)
{
	struct cpt_vf *cptvf = (struct cpt_vf *)vfdev;
	struct pci_dev *pdev = cptvf->pdev;

	if (!cpt_device_ready(cptvf)) {
		dev_err(&pdev->dev, "CPT Device is not ready");
		return -ENODEV;
	}

	if ((cptvf->vftype == SE_TYPES) && (!req->ctrl.s.se_req)) {
		dev_err(&pdev->dev, "CPTVF-%d of SE TYPE got AE request",
			cptvf->vfid);
		return -EINVAL;
	} else if ((cptvf->vftype == AE_TYPES) && (req->ctrl.s.se_req)) {
		dev_err(&pdev->dev, "CPTVF-%d of AE TYPE got SE request",
			cptvf->vfid);
		return -EINVAL;
	}

	return process_request(cptvf, req);
}
Commit	Line	Data
25763b3c	1	// SPDX-License-Identifier: GPL-2.0-only
c694b233 GC	2	/*
c694b233 GC	3	* Copyright (C) 2016 Cavium, Inc.
c694b233 GC	4	*/
	5
	6	#include "cptvf.h"
c4149431	7	#include "cptvf_algs.h"
c694b233 GC	8	#include "request_manager.h"
	9
	10	/**
	11	* get_free_pending_entry - get free entry from pending queue
	12	* @param pqinfo: pending_qinfo structure
	13	* @param qno: queue number
	14	*/
	15	static struct pending_entry get_free_pending_entry(struct pending_queue q,
	16	int qlen)
	17	{
	18	struct pending_entry *ent = NULL;
	19
	20	ent = &q->head[q->rear];
	21	if (unlikely(ent->busy)) {
	22	ent = NULL;
	23	goto no_free_entry;
	24	}
	25
	26	q->rear++;
	27	if (unlikely(q->rear == qlen))
	28	q->rear = 0;
	29
	30	no_free_entry:
	31	return ent;
	32	}
	33
	34	static inline void pending_queue_inc_front(struct pending_qinfo *pqinfo,
	35	int qno)
	36	{
	37	struct pending_queue *queue = &pqinfo->queue[qno];
	38
	39	queue->front++;
	40	if (unlikely(queue->front == pqinfo->qlen))
	41	queue->front = 0;
	42	}
	43
	44	static int setup_sgio_components(struct cpt_vf cptvf, struct buf_ptr list,
	45	int buf_count, u8 *buffer)
	46	{
	47	int ret = 0, i, j;
	48	int components;
	49	struct sglist_component *sg_ptr = NULL;
	50	struct pci_dev *pdev = cptvf->pdev;
	51
	52	if (unlikely(!list)) {
	53	dev_err(&pdev->dev, "Input List pointer is NULL\n");
	54	return -EFAULT;
	55	}
	56
	57	for (i = 0; i < buf_count; i++) {
	58	if (likely(list[i].vptr)) {
	59	list[i].dma_addr = dma_map_single(&pdev->dev,
	60	list[i].vptr,
	61	list[i].size,
	62	DMA_BIDIRECTIONAL);
	63	if (unlikely(dma_mapping_error(&pdev->dev,
	64	list[i].dma_addr))) {
	65	dev_err(&pdev->dev, "DMA map kernel buffer failed for component: %d\n",
	66	i);
	67	ret = -EIO;
	68	goto sg_cleanup;
	69	}
	70	}
	71	}
72
73	components = buf_count / 4;
74	sg_ptr = (struct sglist_component *)buffer;
75	for (i = 0; i < components; i++) {
76	sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
77	sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
78	sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
79	sg_ptr->u.s.len3 = cpu_to_be16(list[i * 4 + 3].size);
80	sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
81	sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
82	sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
83	sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr);
84	sg_ptr++;
85	}
86
87	components = buf_count % 4;
88
89	switch (components) {
90	case 3:
91	sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
92	sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
df561f66	93	fallthrough;
c694b233 GC	94	case 2:
	95	sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
	96	sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
df561f66	97	fallthrough;
c694b233 GC	98	case 1:
	99	sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
	100	sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
	101	break;
	102	default:
	103	break;
	104	}
	105
	106	return ret;
	107
	108	sg_cleanup:
	109	for (j = 0; j < i; j++) {
	110	if (list[j].dma_addr) {
	111	dma_unmap_single(&pdev->dev, list[i].dma_addr,
	112	list[i].size, DMA_BIDIRECTIONAL);
	113	}
	114
	115	list[j].dma_addr = 0;
	116	}
	117
	118	return ret;
	119	}
	120
	121	static inline int setup_sgio_list(struct cpt_vf *cptvf,
	122	struct cpt_info_buffer *info,
	123	struct cpt_request_info *req)
	124	{
	125	u16 g_sz_bytes = 0, s_sz_bytes = 0;
	126	int ret = 0;
	127	struct pci_dev *pdev = cptvf->pdev;
	128
	129	if (req->incnt > MAX_SG_IN_CNT \|\| req->outcnt > MAX_SG_OUT_CNT) {
	130	dev_err(&pdev->dev, "Request SG components are higher than supported\n");
	131	ret = -EINVAL;
	132	goto scatter_gather_clean;
	133	}
	134
	135	/* Setup gather (input) components */
	136	g_sz_bytes = ((req->incnt + 3) / 4) * sizeof(struct sglist_component);
9e27c991	137	info->gather_components = kzalloc(g_sz_bytes, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
c694b233 GC	138	if (!info->gather_components) {
	139	ret = -ENOMEM;
	140	goto scatter_gather_clean;
	141	}
	142
	143	ret = setup_sgio_components(cptvf, req->in,
	144	req->incnt,
	145	info->gather_components);
	146	if (ret) {
	147	dev_err(&pdev->dev, "Failed to setup gather list\n");
	148	ret = -EFAULT;
	149	goto scatter_gather_clean;
	150	}
	151
	152	/* Setup scatter (output) components */
	153	s_sz_bytes = ((req->outcnt + 3) / 4) * sizeof(struct sglist_component);
9e27c991	154	info->scatter_components = kzalloc(s_sz_bytes, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
c694b233 GC	155	if (!info->scatter_components) {
	156	ret = -ENOMEM;
	157	goto scatter_gather_clean;
	158	}
	159
	160	ret = setup_sgio_components(cptvf, req->out,
	161	req->outcnt,
	162	info->scatter_components);
	163	if (ret) {
	164	dev_err(&pdev->dev, "Failed to setup gather list\n");
	165	ret = -EFAULT;
	166	goto scatter_gather_clean;
	167	}
	168
	169	/* Create and initialize DPTR */
	170	info->dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE;
9e27c991	171	info->in_buffer = kzalloc(info->dlen, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
c694b233 GC	172	if (!info->in_buffer) {
	173	ret = -ENOMEM;
	174	goto scatter_gather_clean;
	175	}
	176
c4149431 HX	177	((__be16 *)info->in_buffer)[0] = cpu_to_be16(req->outcnt);
	178	((__be16 *)info->in_buffer)[1] = cpu_to_be16(req->incnt);
	179	((__be16 *)info->in_buffer)[2] = 0;
	180	((__be16 *)info->in_buffer)[3] = 0;
c694b233 GC	181
	182	memcpy(&info->in_buffer[8], info->gather_components,
	183	g_sz_bytes);
	184	memcpy(&info->in_buffer[8 + g_sz_bytes],
	185	info->scatter_components, s_sz_bytes);
	186
	187	info->dptr_baddr = dma_map_single(&pdev->dev,
	188	(void *)info->in_buffer,
	189	info->dlen,
	190	DMA_BIDIRECTIONAL);
	191	if (dma_mapping_error(&pdev->dev, info->dptr_baddr)) {
	192	dev_err(&pdev->dev, "Mapping DPTR Failed %d\n", info->dlen);
	193	ret = -EIO;
	194	goto scatter_gather_clean;
	195	}
	196
	197	/* Create and initialize RPTR */
9e27c991	198	info->out_buffer = kzalloc(COMPLETION_CODE_SIZE, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
c694b233 GC	199	if (!info->out_buffer) {
	200	ret = -ENOMEM;
	201	goto scatter_gather_clean;
	202	}
	203
	204	((u64 )info->out_buffer) = ~((u64)COMPLETION_CODE_INIT);
	205	info->alternate_caddr = (u64 *)info->out_buffer;
	206	info->rptr_baddr = dma_map_single(&pdev->dev,
	207	(void *)info->out_buffer,
	208	COMPLETION_CODE_SIZE,
	209	DMA_BIDIRECTIONAL);
	210	if (dma_mapping_error(&pdev->dev, info->rptr_baddr)) {
	211	dev_err(&pdev->dev, "Mapping RPTR Failed %d\n",
	212	COMPLETION_CODE_SIZE);
	213	ret = -EIO;
	214	goto scatter_gather_clean;
	215	}
	216
	217	return 0;
	218
	219	scatter_gather_clean:
	220	return ret;
	221	}
	222
cd1af982	223	static int send_cpt_command(struct cpt_vf cptvf, union cpt_inst_s cmd,
c694b233 GC	224	u32 qno)
	225	{
	226	struct pci_dev *pdev = cptvf->pdev;
	227	struct command_qinfo *qinfo = NULL;
	228	struct command_queue *queue;
	229	struct command_chunk *chunk;
	230	u8 *ent;
	231	int ret = 0;
	232
	233	if (unlikely(qno >= cptvf->nr_queues)) {
	234	dev_err(&pdev->dev, "Invalid queue (qno: %d, nr_queues: %d)\n",
	235	qno, cptvf->nr_queues);
	236	return -EINVAL;
	237	}
	238
	239	qinfo = &cptvf->cqinfo;
	240	queue = &qinfo->queue[qno];
	241	/* lock commad queue */
	242	spin_lock(&queue->lock);
	243	ent = &queue->qhead->head[queue->idx * qinfo->cmd_size];
	244	memcpy(ent, (void *)cmd, qinfo->cmd_size);
	245
	246	if (++queue->idx >= queue->qhead->size / 64) {
	247	struct hlist_node *node;
	248
	249	hlist_for_each(node, &queue->chead) {
	250	chunk = hlist_entry(node, struct command_chunk,
	251	nextchunk);
	252	if (chunk == queue->qhead) {
	253	continue;
	254	} else {
	255	queue->qhead = chunk;
	256	break;
	257	}
	258	}
	259	queue->idx = 0;
	260	}
	261	/* make sure all memory stores are done before ringing doorbell */
	262	smp_wmb();
	263	cptvf_write_vq_doorbell(cptvf, 1);
	264	/* unlock command queue */
	265	spin_unlock(&queue->lock);
	266
	267	return ret;
	268	}
	269
cd1af982	270	static void do_request_cleanup(struct cpt_vf *cptvf,
c694b233 GC	271	struct cpt_info_buffer *info)
	272	{
	273	int i;
	274	struct pci_dev *pdev = cptvf->pdev;
	275	struct cpt_request_info *req;
	276
	277	if (info->dptr_baddr)
	278	dma_unmap_single(&pdev->dev, info->dptr_baddr,
	279	info->dlen, DMA_BIDIRECTIONAL);
	280
	281	if (info->rptr_baddr)
	282	dma_unmap_single(&pdev->dev, info->rptr_baddr,
	283	COMPLETION_CODE_SIZE, DMA_BIDIRECTIONAL);
	284
	285	if (info->comp_baddr)
	286	dma_unmap_single(&pdev->dev, info->comp_baddr,
	287	sizeof(union cpt_res_s), DMA_BIDIRECTIONAL);
	288
	289	if (info->req) {
	290	req = info->req;
	291	for (i = 0; i < req->outcnt; i++) {
	292	if (req->out[i].dma_addr)
	293	dma_unmap_single(&pdev->dev,
	294	req->out[i].dma_addr,
	295	req->out[i].size,
	296	DMA_BIDIRECTIONAL);
	297	}
	298
	299	for (i = 0; i < req->incnt; i++) {
	300	if (req->in[i].dma_addr)
	301	dma_unmap_single(&pdev->dev,
	302	req->in[i].dma_addr,
	303	req->in[i].size,
	304	DMA_BIDIRECTIONAL);
	305	}
	306	}
	307
453431a5 WL	308	kfree_sensitive(info->scatter_components);
	309	kfree_sensitive(info->gather_components);
	310	kfree_sensitive(info->out_buffer);
	311	kfree_sensitive(info->in_buffer);
	312	kfree_sensitive((void *)info->completion_addr);
	313	kfree_sensitive(info);
c694b233 GC	314	}
c694b233 GC	315
cd1af982	316	static void do_post_process(struct cpt_vf cptvf, struct cpt_info_buffer info)
c694b233 GC	317	{
	318	struct pci_dev *pdev = cptvf->pdev;
	319
9bd82904 GC	320	if (!info) {
9bd82904 GC	321	dev_err(&pdev->dev, "incorrect cpt_info_buffer for post processing\n");
c694b233 GC	322	return;
	323	}
	324
	325	do_request_cleanup(cptvf, info);
	326	}
	327
	328	static inline void process_pending_queue(struct cpt_vf *cptvf,
	329	struct pending_qinfo *pqinfo,
	330	int qno)
	331	{
	332	struct pci_dev *pdev = cptvf->pdev;
	333	struct pending_queue *pqueue = &pqinfo->queue[qno];
	334	struct pending_entry *pentry = NULL;
	335	struct cpt_info_buffer *info = NULL;
	336	union cpt_res_s *status = NULL;
	337	unsigned char ccode;
	338
	339	while (1) {
	340	spin_lock_bh(&pqueue->lock);
	341	pentry = &pqueue->head[pqueue->front];
	342	if (unlikely(!pentry->busy)) {
	343	spin_unlock_bh(&pqueue->lock);
	344	break;
	345	}
	346
	347	info = (struct cpt_info_buffer *)pentry->post_arg;
	348	if (unlikely(!info)) {
	349	dev_err(&pdev->dev, "Pending Entry post arg NULL\n");
	350	pending_queue_inc_front(pqinfo, qno);
	351	spin_unlock_bh(&pqueue->lock);
	352	continue;
	353	}
	354
	355	status = (union cpt_res_s *)pentry->completion_addr;
	356	ccode = status->s.compcode;
	357	if ((status->s.compcode == CPT_COMP_E_FAULT) \|\|
	358	(status->s.compcode == CPT_COMP_E_SWERR)) {
	359	dev_err(&pdev->dev, "Request failed with %s\n",
	360	(status->s.compcode == CPT_COMP_E_FAULT) ?
	361	"DMA Fault" : "Software error");
	362	pentry->completion_addr = NULL;
	363	pentry->busy = false;
	364	atomic64_dec((&pqueue->pending_count));
	365	pentry->post_arg = NULL;
	366	pending_queue_inc_front(pqinfo, qno);
	367	do_request_cleanup(cptvf, info);
	368	spin_unlock_bh(&pqueue->lock);
	369	break;
	370	} else if (status->s.compcode == COMPLETION_CODE_INIT) {
	371	/* check for timeout */
	372	if (time_after_eq(jiffies,
	373	(info->time_in +
	374	(CPT_COMMAND_TIMEOUT * HZ)))) {
	375	dev_err(&pdev->dev, "Request timed out");
	376	pentry->completion_addr = NULL;
	377	pentry->busy = false;
	378	atomic64_dec((&pqueue->pending_count));
	379	pentry->post_arg = NULL;
	380	pending_queue_inc_front(pqinfo, qno);
	381	do_request_cleanup(cptvf, info);
	382	spin_unlock_bh(&pqueue->lock);
	383	break;
	384	} else if ((*info->alternate_caddr ==
	385	(~COMPLETION_CODE_INIT)) &&
386	(info->extra_time < TIME_IN_RESET_COUNT)) {
387	info->time_in = jiffies;
388	info->extra_time++;
389	spin_unlock_bh(&pqueue->lock);
390	break;
391	}
392	}
393
394	pentry->completion_addr = NULL;
395	pentry->busy = false;
396	pentry->post_arg = NULL;
397	atomic64_dec((&pqueue->pending_count));
398	pending_queue_inc_front(pqinfo, qno);
399	spin_unlock_bh(&pqueue->lock);
400
401	do_post_process(info->cptvf, info);
402	/*
403	* Calling callback after we find
404	* that the request has been serviced
405	*/
406	pentry->callback(ccode, pentry->callback_arg);
407	}
408	}
409
410	int process_request(struct cpt_vf cptvf, struct cpt_request_info req)
411	{
412	int ret = 0, clear = 0, queue = 0;
413	struct cpt_info_buffer *info = NULL;
414	struct cptvf_request *cpt_req = NULL;
415	union ctrl_info *ctrl = NULL;
416	union cpt_res_s *result = NULL;
417	struct pending_entry *pentry = NULL;
418	struct pending_queue *pqueue = NULL;
419	struct pci_dev *pdev = cptvf->pdev;
420	u8 group = 0;
421	struct cpt_vq_command vq_cmd;
422	union cpt_inst_s cptinst;
423
9e27c991	424	info = kzalloc(sizeof(*info), req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
c694b233 GC	425	if (unlikely(!info)) {
	426	dev_err(&pdev->dev, "Unable to allocate memory for info_buffer\n");
	427	return -ENOMEM;
	428	}
	429
	430	cpt_req = (struct cptvf_request *)&req->req;
	431	ctrl = (union ctrl_info *)&req->ctrl;
	432
	433	info->cptvf = cptvf;
	434	group = ctrl->s.grp;
	435	ret = setup_sgio_list(cptvf, info, req);
	436	if (ret) {
	437	dev_err(&pdev->dev, "Setting up SG list failed");
	438	goto request_cleanup;
	439	}
	440
	441	cpt_req->dlen = info->dlen;
	442	/*
	443	* Get buffer for union cpt_res_s response
	444	* structure and its physical address
	445	*/
9e27c991	446	info->completion_addr = kzalloc(sizeof(union cpt_res_s), req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
c694b233 GC	447	if (unlikely(!info->completion_addr)) {
c694b233 GC	448	dev_err(&pdev->dev, "Unable to allocate memory for completion_addr\n");
87aae50a CIK	449	ret = -ENOMEM;
87aae50a CIK	450	goto request_cleanup;
c694b233 GC	451	}
	452
	453	result = (union cpt_res_s *)info->completion_addr;
	454	result->s.compcode = COMPLETION_CODE_INIT;
	455	info->comp_baddr = dma_map_single(&pdev->dev,
	456	(void *)info->completion_addr,
	457	sizeof(union cpt_res_s),
	458	DMA_BIDIRECTIONAL);
	459	if (dma_mapping_error(&pdev->dev, info->comp_baddr)) {
	460	dev_err(&pdev->dev, "mapping compptr Failed %lu\n",
	461	sizeof(union cpt_res_s));
	462	ret = -EFAULT;
	463	goto request_cleanup;
	464	}
	465
	466	/* Fill the VQ command */
	467	vq_cmd.cmd.u64 = 0;
	468	vq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags);
	469	vq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1);
	470	vq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2);
	471	vq_cmd.cmd.s.dlen = cpu_to_be16(cpt_req->dlen);
	472
c694b233 GC	473	vq_cmd.dptr = info->dptr_baddr;
	474	vq_cmd.rptr = info->rptr_baddr;
	475	vq_cmd.cptr.u64 = 0;
	476	vq_cmd.cptr.s.grp = group;
	477	/* Get Pending Entry to submit command */
	478	/* Always queue 0, because 1 queue per VF */
	479	queue = 0;
	480	pqueue = &cptvf->pqinfo.queue[queue];
	481
	482	if (atomic64_read(&pqueue->pending_count) > PENDING_THOLD) {
	483	dev_err(&pdev->dev, "pending threshold reached\n");
	484	process_pending_queue(cptvf, &cptvf->pqinfo, queue);
	485	}
	486
	487	get_pending_entry:
	488	spin_lock_bh(&pqueue->lock);
	489	pentry = get_free_pending_entry(pqueue, cptvf->pqinfo.qlen);
	490	if (unlikely(!pentry)) {
	491	spin_unlock_bh(&pqueue->lock);
	492	if (clear == 0) {
	493	process_pending_queue(cptvf, &cptvf->pqinfo, queue);
	494	clear = 1;
	495	goto get_pending_entry;
	496	}
	497	dev_err(&pdev->dev, "Get free entry failed\n");
	498	dev_err(&pdev->dev, "queue: %d, rear: %d, front: %d\n",
	499	queue, pqueue->rear, pqueue->front);
	500	ret = -EFAULT;
	501	goto request_cleanup;
	502	}
	503
	504	pentry->completion_addr = info->completion_addr;
	505	pentry->post_arg = (void *)info;
	506	pentry->callback = req->callback;
	507	pentry->callback_arg = req->callback_arg;
	508	info->pentry = pentry;
	509	pentry->busy = true;
	510	atomic64_inc(&pqueue->pending_count);
	511
	512	/* Send CPT command */
	513	info->pentry = pentry;
	514	info->time_in = jiffies;
	515	info->req = req;
	516
	517	/* Create the CPT_INST_S type command for HW intrepretation */
	518	cptinst.s.doneint = true;
	519	cptinst.s.res_addr = (u64)info->comp_baddr;
	520	cptinst.s.tag = 0;
	521	cptinst.s.grp = 0;
	522	cptinst.s.wq_ptr = 0;
	523	cptinst.s.ei0 = vq_cmd.cmd.u64;
	524	cptinst.s.ei1 = vq_cmd.dptr;
	525	cptinst.s.ei2 = vq_cmd.rptr;
	526	cptinst.s.ei3 = vq_cmd.cptr.u64;
	527
	528	ret = send_cpt_command(cptvf, &cptinst, queue);
	529	spin_unlock_bh(&pqueue->lock);
	530	if (unlikely(ret)) {
	531	dev_err(&pdev->dev, "Send command failed for AE\n");
	532	ret = -EFAULT;
	533	goto request_cleanup;
	534	}
	535
	536	return 0;
537
538	request_cleanup:
539	dev_dbg(&pdev->dev, "Failed to submit CPT command\n");
540	do_request_cleanup(cptvf, info);
541
542	return ret;
543	}
544
545	void vq_post_process(struct cpt_vf *cptvf, u32 qno)
546	{
547	struct pci_dev *pdev = cptvf->pdev;
548
549	if (unlikely(qno > cptvf->nr_queues)) {
550	dev_err(&pdev->dev, "Request for post processing on invalid pending queue: %u\n",
551	qno);
552	return;
553	}
554
555	process_pending_queue(cptvf, &cptvf->pqinfo, qno);
556	}
557
558	int cptvf_do_request(void vfdev, struct cpt_request_info req)
559	{
560	struct cpt_vf cptvf = (struct cpt_vf )vfdev;
561	struct pci_dev *pdev = cptvf->pdev;
562
563	if (!cpt_device_ready(cptvf)) {
564	dev_err(&pdev->dev, "CPT Device is not ready");
565	return -ENODEV;
566	}
567
568	if ((cptvf->vftype == SE_TYPES) && (!req->ctrl.s.se_req)) {
569	dev_err(&pdev->dev, "CPTVF-%d of SE TYPE got AE request",
570	cptvf->vfid);
571	return -EINVAL;
572	} else if ((cptvf->vftype == AE_TYPES) && (req->ctrl.s.se_req)) {
573	dev_err(&pdev->dev, "CPTVF-%d of AE TYPE got SE request",
574	cptvf->vfid);
575	return -EINVAL;
576	}
577
578	return process_request(cptvf, req);
579	}