[mirror_ubuntu-focal-kernel.git] / drivers / net / ethernet / huawei / hinic / hinic_hw_io.c

/*
 * Huawei HiNIC PCI Express Linux driver
 * Copyright(c) 2017 Huawei Technologies Co., Ltd
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 */

#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/pci.h>
#include <linux/device.h>
#include <linux/errno.h>
#include <linux/slab.h>
#include <linux/semaphore.h>
#include <linux/dma-mapping.h>
#include <linux/io.h>
#include <linux/err.h>

#include "hinic_hw_if.h"
#include "hinic_hw_eqs.h"
#include "hinic_hw_wqe.h"
#include "hinic_hw_wq.h"
#include "hinic_hw_cmdq.h"
#include "hinic_hw_qp_ctxt.h"
#include "hinic_hw_qp.h"
#include "hinic_hw_io.h"

#define CI_Q_ADDR_SIZE                  sizeof(u32)

#define CI_ADDR(base_addr, q_id)        ((base_addr) + \
					 (q_id) * CI_Q_ADDR_SIZE)

#define CI_TABLE_SIZE(num_qps)          ((num_qps) * CI_Q_ADDR_SIZE)

#define DB_IDX(db, db_base)             \
	(((unsigned long)(db) - (unsigned long)(db_base)) / HINIC_DB_PAGE_SIZE)

enum io_cmd {
	IO_CMD_MODIFY_QUEUE_CTXT = 0,
};

static void init_db_area_idx(struct hinic_free_db_area *free_db_area)
{
	int i;

	for (i = 0; i < HINIC_DB_MAX_AREAS; i++)
		free_db_area->db_idx[i] = i;

	free_db_area->alloc_pos = 0;
	free_db_area->return_pos = HINIC_DB_MAX_AREAS;

	free_db_area->num_free = HINIC_DB_MAX_AREAS;

	sema_init(&free_db_area->idx_lock, 1);
}

static void __iomem *get_db_area(struct hinic_func_to_io *func_to_io)
{
	struct hinic_free_db_area *free_db_area = &func_to_io->free_db_area;
	int pos, idx;

	down(&free_db_area->idx_lock);

	free_db_area->num_free--;

	if (free_db_area->num_free < 0) {
		free_db_area->num_free++;
		up(&free_db_area->idx_lock);
		return ERR_PTR(-ENOMEM);
	}

	pos = free_db_area->alloc_pos++;
	pos &= HINIC_DB_MAX_AREAS - 1;

	idx = free_db_area->db_idx[pos];

	free_db_area->db_idx[pos] = -1;

	up(&free_db_area->idx_lock);

	return func_to_io->db_base + idx * HINIC_DB_PAGE_SIZE;
}

static void return_db_area(struct hinic_func_to_io *func_to_io,
			   void __iomem *db_base)
{
	struct hinic_free_db_area *free_db_area = &func_to_io->free_db_area;
	int pos, idx = DB_IDX(db_base, func_to_io->db_base);

	down(&free_db_area->idx_lock);

	pos = free_db_area->return_pos++;
	pos &= HINIC_DB_MAX_AREAS - 1;

	free_db_area->db_idx[pos] = idx;

	free_db_area->num_free++;

	up(&free_db_area->idx_lock);
}

static int write_sq_ctxts(struct hinic_func_to_io *func_to_io, u16 base_qpn,
			  u16 num_sqs)
{
	struct hinic_hwif *hwif = func_to_io->hwif;
	struct hinic_sq_ctxt_block *sq_ctxt_block;
	struct pci_dev *pdev = hwif->pdev;
	struct hinic_cmdq_buf cmdq_buf;
	struct hinic_sq_ctxt *sq_ctxt;
	struct hinic_qp *qp;
	u64 out_param;
	int err, i;

	err = hinic_alloc_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf);
	if (err) {
		dev_err(&pdev->dev, "Failed to allocate cmdq buf\n");
		return err;
	}

	sq_ctxt_block = cmdq_buf.buf;
	sq_ctxt = sq_ctxt_block->sq_ctxt;

	hinic_qp_prepare_header(&sq_ctxt_block->hdr, HINIC_QP_CTXT_TYPE_SQ,
				num_sqs, func_to_io->max_qps);
	for (i = 0; i < num_sqs; i++) {
		qp = &func_to_io->qps[i];

		hinic_sq_prepare_ctxt(&sq_ctxt[i], &qp->sq,
				      base_qpn + qp->q_id);
	}

	cmdq_buf.size = HINIC_SQ_CTXT_SIZE(num_sqs);

	err = hinic_cmdq_direct_resp(&func_to_io->cmdqs, HINIC_MOD_L2NIC,
				     IO_CMD_MODIFY_QUEUE_CTXT, &cmdq_buf,
				     &out_param);
	if ((err) || (out_param != 0)) {
		dev_err(&pdev->dev, "Failed to set SQ ctxts\n");
		err = -EFAULT;
	}

	hinic_free_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf);
	return err;
}

static int write_rq_ctxts(struct hinic_func_to_io *func_to_io, u16 base_qpn,
			  u16 num_rqs)
{
	struct hinic_hwif *hwif = func_to_io->hwif;
	struct hinic_rq_ctxt_block *rq_ctxt_block;
	struct pci_dev *pdev = hwif->pdev;
	struct hinic_cmdq_buf cmdq_buf;
	struct hinic_rq_ctxt *rq_ctxt;
	struct hinic_qp *qp;
	u64 out_param;
	int err, i;

	err = hinic_alloc_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf);
	if (err) {
		dev_err(&pdev->dev, "Failed to allocate cmdq buf\n");
		return err;
	}

	rq_ctxt_block = cmdq_buf.buf;
	rq_ctxt = rq_ctxt_block->rq_ctxt;

	hinic_qp_prepare_header(&rq_ctxt_block->hdr, HINIC_QP_CTXT_TYPE_RQ,
				num_rqs, func_to_io->max_qps);
	for (i = 0; i < num_rqs; i++) {
		qp = &func_to_io->qps[i];

		hinic_rq_prepare_ctxt(&rq_ctxt[i], &qp->rq,
				      base_qpn + qp->q_id);
	}

	cmdq_buf.size = HINIC_RQ_CTXT_SIZE(num_rqs);

	err = hinic_cmdq_direct_resp(&func_to_io->cmdqs, HINIC_MOD_L2NIC,
				     IO_CMD_MODIFY_QUEUE_CTXT, &cmdq_buf,
				     &out_param);
	if ((err) || (out_param != 0)) {
		dev_err(&pdev->dev, "Failed to set RQ ctxts\n");
		err = -EFAULT;
	}

	hinic_free_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf);
	return err;
}

/**
 * write_qp_ctxts - write the qp ctxt to HW
 * @func_to_io: func to io channel that holds the IO components
 * @base_qpn: first qp number
 * @num_qps: number of qps to write
 *
 * Return 0 - Success, negative - Failure
 **/
static int write_qp_ctxts(struct hinic_func_to_io *func_to_io, u16 base_qpn,
			  u16 num_qps)
{
	return (write_sq_ctxts(func_to_io, base_qpn, num_qps) ||
		write_rq_ctxts(func_to_io, base_qpn, num_qps));
}

/**
 * init_qp - Initialize a Queue Pair
 * @func_to_io: func to io channel that holds the IO components
 * @qp: pointer to the qp to initialize
 * @q_id: the id of the qp
 * @sq_msix_entry: msix entry for sq
 * @rq_msix_entry: msix entry for rq
 *
 * Return 0 - Success, negative - Failure
 **/
static int init_qp(struct hinic_func_to_io *func_to_io,
		   struct hinic_qp *qp, int q_id,
		   struct msix_entry *sq_msix_entry,
		   struct msix_entry *rq_msix_entry)
{
	struct hinic_hwif *hwif = func_to_io->hwif;
	struct pci_dev *pdev = hwif->pdev;
	void __iomem *db_base;
	int err;

	qp->q_id = q_id;

	err = hinic_wq_allocate(&func_to_io->wqs, &func_to_io->sq_wq[q_id],
				HINIC_SQ_WQEBB_SIZE, HINIC_SQ_PAGE_SIZE,
				HINIC_SQ_DEPTH, HINIC_SQ_WQE_MAX_SIZE);
	if (err) {
		dev_err(&pdev->dev, "Failed to allocate WQ for SQ\n");
		return err;
	}

	err = hinic_wq_allocate(&func_to_io->wqs, &func_to_io->rq_wq[q_id],
				HINIC_RQ_WQEBB_SIZE, HINIC_RQ_PAGE_SIZE,
				HINIC_RQ_DEPTH, HINIC_RQ_WQE_SIZE);
	if (err) {
		dev_err(&pdev->dev, "Failed to allocate WQ for RQ\n");
		goto err_rq_alloc;
	}

	db_base = get_db_area(func_to_io);
	if (IS_ERR(db_base)) {
		dev_err(&pdev->dev, "Failed to get DB area for SQ\n");
		err = PTR_ERR(db_base);
		goto err_get_db;
	}

	func_to_io->sq_db[q_id] = db_base;

	err = hinic_init_sq(&qp->sq, hwif, &func_to_io->sq_wq[q_id],
			    sq_msix_entry,
			    CI_ADDR(func_to_io->ci_addr_base, q_id),
			    CI_ADDR(func_to_io->ci_dma_base, q_id), db_base);
	if (err) {
		dev_err(&pdev->dev, "Failed to init SQ\n");
		goto err_sq_init;
	}

	err = hinic_init_rq(&qp->rq, hwif, &func_to_io->rq_wq[q_id],
			    rq_msix_entry);
	if (err) {
		dev_err(&pdev->dev, "Failed to init RQ\n");
		goto err_rq_init;
	}

	return 0;

err_rq_init:
	hinic_clean_sq(&qp->sq);

err_sq_init:
	return_db_area(func_to_io, db_base);

err_get_db:
	hinic_wq_free(&func_to_io->wqs, &func_to_io->rq_wq[q_id]);

err_rq_alloc:
	hinic_wq_free(&func_to_io->wqs, &func_to_io->sq_wq[q_id]);
	return err;
}

/**
 * destroy_qp - Clean the resources of a Queue Pair
 * @func_to_io: func to io channel that holds the IO components
 * @qp: pointer to the qp to clean
 **/
static void destroy_qp(struct hinic_func_to_io *func_to_io,
		       struct hinic_qp *qp)
{
	int q_id = qp->q_id;

	hinic_clean_rq(&qp->rq);
	hinic_clean_sq(&qp->sq);

	return_db_area(func_to_io, func_to_io->sq_db[q_id]);

	hinic_wq_free(&func_to_io->wqs, &func_to_io->rq_wq[q_id]);
	hinic_wq_free(&func_to_io->wqs, &func_to_io->sq_wq[q_id]);
}

/**
 * hinic_io_create_qps - Create Queue Pairs
 * @func_to_io: func to io channel that holds the IO components
 * @base_qpn: base qp number
 * @num_qps: number queue pairs to create
 * @sq_msix_entry: msix entries for sq
 * @rq_msix_entry: msix entries for rq
 *
 * Return 0 - Success, negative - Failure
 **/
int hinic_io_create_qps(struct hinic_func_to_io *func_to_io,
			u16 base_qpn, int num_qps,
			struct msix_entry *sq_msix_entries,
			struct msix_entry *rq_msix_entries)
{
	struct hinic_hwif *hwif = func_to_io->hwif;
	struct pci_dev *pdev = hwif->pdev;
	size_t qps_size, wq_size, db_size;
	void *ci_addr_base;
	int i, j, err;

	qps_size = num_qps * sizeof(*func_to_io->qps);
	func_to_io->qps = devm_kzalloc(&pdev->dev, qps_size, GFP_KERNEL);
	if (!func_to_io->qps)
		return -ENOMEM;

	wq_size = num_qps * sizeof(*func_to_io->sq_wq);
	func_to_io->sq_wq = devm_kzalloc(&pdev->dev, wq_size, GFP_KERNEL);
	if (!func_to_io->sq_wq) {
		err = -ENOMEM;
		goto err_sq_wq;
	}

	wq_size = num_qps * sizeof(*func_to_io->rq_wq);
	func_to_io->rq_wq = devm_kzalloc(&pdev->dev, wq_size, GFP_KERNEL);
	if (!func_to_io->rq_wq) {
		err = -ENOMEM;
		goto err_rq_wq;
	}

	db_size = num_qps * sizeof(*func_to_io->sq_db);
	func_to_io->sq_db = devm_kzalloc(&pdev->dev, db_size, GFP_KERNEL);
	if (!func_to_io->sq_db) {
		err = -ENOMEM;
		goto err_sq_db;
	}

	ci_addr_base = dma_alloc_coherent(&pdev->dev, CI_TABLE_SIZE(num_qps),
					  &func_to_io->ci_dma_base,
					  GFP_KERNEL);
	if (!ci_addr_base) {
		dev_err(&pdev->dev, "Failed to allocate CI area\n");
		err = -ENOMEM;
		goto err_ci_base;
	}

	func_to_io->ci_addr_base = ci_addr_base;

	for (i = 0; i < num_qps; i++) {
		err = init_qp(func_to_io, &func_to_io->qps[i], i,
			      &sq_msix_entries[i], &rq_msix_entries[i]);
		if (err) {
			dev_err(&pdev->dev, "Failed to create QP %d\n", i);
			goto err_init_qp;
		}
	}

	err = write_qp_ctxts(func_to_io, base_qpn, num_qps);
	if (err) {
		dev_err(&pdev->dev, "Failed to init QP ctxts\n");
		goto err_write_qp_ctxts;
	}

	return 0;

err_write_qp_ctxts:
err_init_qp:
	for (j = 0; j < i; j++)
		destroy_qp(func_to_io, &func_to_io->qps[j]);

	dma_free_coherent(&pdev->dev, CI_TABLE_SIZE(num_qps),
			  func_to_io->ci_addr_base, func_to_io->ci_dma_base);

err_ci_base:
	devm_kfree(&pdev->dev, func_to_io->sq_db);

err_sq_db:
	devm_kfree(&pdev->dev, func_to_io->rq_wq);

err_rq_wq:
	devm_kfree(&pdev->dev, func_to_io->sq_wq);

err_sq_wq:
	devm_kfree(&pdev->dev, func_to_io->qps);
	return err;
}

/**
 * hinic_io_destroy_qps - Destroy the IO Queue Pairs
 * @func_to_io: func to io channel that holds the IO components
 * @num_qps: number queue pairs to destroy
 **/
void hinic_io_destroy_qps(struct hinic_func_to_io *func_to_io, int num_qps)
{
	struct hinic_hwif *hwif = func_to_io->hwif;
	struct pci_dev *pdev = hwif->pdev;
	size_t ci_table_size;
	int i;

	ci_table_size = CI_TABLE_SIZE(num_qps);

	for (i = 0; i < num_qps; i++)
		destroy_qp(func_to_io, &func_to_io->qps[i]);

	dma_free_coherent(&pdev->dev, ci_table_size, func_to_io->ci_addr_base,
			  func_to_io->ci_dma_base);

	devm_kfree(&pdev->dev, func_to_io->sq_db);

	devm_kfree(&pdev->dev, func_to_io->rq_wq);
	devm_kfree(&pdev->dev, func_to_io->sq_wq);

	devm_kfree(&pdev->dev, func_to_io->qps);
}

/**
 * hinic_io_init - Initialize the IO components
 * @func_to_io: func to io channel that holds the IO components
 * @hwif: HW interface for accessing IO
 * @max_qps: maximum QPs in HW
 * @num_ceqs: number completion event queues
 * @ceq_msix_entries: msix entries for ceqs
 *
 * Return 0 - Success, negative - Failure
 **/
int hinic_io_init(struct hinic_func_to_io *func_to_io,
		  struct hinic_hwif *hwif, u16 max_qps, int num_ceqs,
		  struct msix_entry *ceq_msix_entries)
{
	struct pci_dev *pdev = hwif->pdev;
	enum hinic_cmdq_type cmdq, type;
	void __iomem *db_area;
	int err;

	func_to_io->hwif = hwif;
	func_to_io->qps = NULL;
	func_to_io->max_qps = max_qps;

	err = hinic_ceqs_init(&func_to_io->ceqs, hwif, num_ceqs,
			      HINIC_DEFAULT_CEQ_LEN, HINIC_EQ_PAGE_SIZE,
			      ceq_msix_entries);
	if (err) {
		dev_err(&pdev->dev, "Failed to init CEQs\n");
		return err;
	}

	err = hinic_wqs_alloc(&func_to_io->wqs, 2 * max_qps, hwif);
	if (err) {
		dev_err(&pdev->dev, "Failed to allocate WQS for IO\n");
		goto err_wqs_alloc;
	}

	func_to_io->db_base = pci_ioremap_bar(pdev, HINIC_PCI_DB_BAR);
	if (!func_to_io->db_base) {
		dev_err(&pdev->dev, "Failed to remap IO DB area\n");
		err = -ENOMEM;
		goto err_db_ioremap;
	}

	init_db_area_idx(&func_to_io->free_db_area);

	for (cmdq = HINIC_CMDQ_SYNC; cmdq < HINIC_MAX_CMDQ_TYPES; cmdq++) {
		db_area = get_db_area(func_to_io);
		if (IS_ERR(db_area)) {
			dev_err(&pdev->dev, "Failed to get cmdq db area\n");
			err = PTR_ERR(db_area);
			goto err_db_area;
		}

		func_to_io->cmdq_db_area[cmdq] = db_area;
	}

	err = hinic_init_cmdqs(&func_to_io->cmdqs, hwif,
			       func_to_io->cmdq_db_area);
	if (err) {
		dev_err(&pdev->dev, "Failed to initialize cmdqs\n");
		goto err_init_cmdqs;
	}

	return 0;

err_init_cmdqs:
err_db_area:
	for (type = HINIC_CMDQ_SYNC; type < cmdq; type++)
		return_db_area(func_to_io, func_to_io->cmdq_db_area[type]);

	iounmap(func_to_io->db_base);

err_db_ioremap:
	hinic_wqs_free(&func_to_io->wqs);

err_wqs_alloc:
	hinic_ceqs_free(&func_to_io->ceqs);
	return err;
}

/**
 * hinic_io_free - Free the IO components
 * @func_to_io: func to io channel that holds the IO components
 **/
void hinic_io_free(struct hinic_func_to_io *func_to_io)
{
	enum hinic_cmdq_type cmdq;

	hinic_free_cmdqs(&func_to_io->cmdqs);

	for (cmdq = HINIC_CMDQ_SYNC; cmdq < HINIC_MAX_CMDQ_TYPES; cmdq++)
		return_db_area(func_to_io, func_to_io->cmdq_db_area[cmdq]);

	iounmap(func_to_io->db_base);
	hinic_wqs_free(&func_to_io->wqs);
	hinic_ceqs_free(&func_to_io->ceqs);
}
Commit	Line	Data
c3e79baf AK	1	/*
	2	* Huawei HiNIC PCI Express Linux driver
	3	* Copyright(c) 2017 Huawei Technologies Co., Ltd
	4	*
	5	* This program is free software; you can redistribute it and/or modify it
	6	* under the terms and conditions of the GNU General Public License,
	7	* version 2, as published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope it will be useful, but WITHOUT
	10	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	11	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
	12	* for more details.
	13	*
	14	*/
	15
f91090f7	16	#include <linux/kernel.h>
c3e79baf AK	17	#include <linux/types.h>
	18	#include <linux/pci.h>
	19	#include <linux/device.h>
	20	#include <linux/errno.h>
	21	#include <linux/slab.h>
f91090f7 AK	22	#include <linux/semaphore.h>
	23	#include <linux/dma-mapping.h>
	24	#include <linux/io.h>
	25	#include <linux/err.h>
c3e79baf AK	26
c3e79baf AK	27	#include "hinic_hw_if.h"
fc9319e4	28	#include "hinic_hw_eqs.h"
b15a9f37 AK	29	#include "hinic_hw_wqe.h"
b15a9f37 AK	30	#include "hinic_hw_wq.h"
53e7d6fe AK	31	#include "hinic_hw_cmdq.h"
53e7d6fe AK	32	#include "hinic_hw_qp_ctxt.h"
c3e79baf AK	33	#include "hinic_hw_qp.h"
	34	#include "hinic_hw_io.h"
	35
f91090f7 AK	36	#define CI_Q_ADDR_SIZE sizeof(u32)
	37
	38	#define CI_ADDR(base_addr, q_id) ((base_addr) + \
	39	(q_id) * CI_Q_ADDR_SIZE)
	40
	41	#define CI_TABLE_SIZE(num_qps) ((num_qps) * CI_Q_ADDR_SIZE)
	42
	43	#define DB_IDX(db, db_base) \
	44	(((unsigned long)(db) - (unsigned long)(db_base)) / HINIC_DB_PAGE_SIZE)
	45
53e7d6fe AK	46	enum io_cmd {
	47	IO_CMD_MODIFY_QUEUE_CTXT = 0,
	48	};
	49
f91090f7 AK	50	static void init_db_area_idx(struct hinic_free_db_area *free_db_area)
	51	{
	52	int i;
	53
	54	for (i = 0; i < HINIC_DB_MAX_AREAS; i++)
	55	free_db_area->db_idx[i] = i;
	56
	57	free_db_area->alloc_pos = 0;
	58	free_db_area->return_pos = HINIC_DB_MAX_AREAS;
	59
	60	free_db_area->num_free = HINIC_DB_MAX_AREAS;
	61
	62	sema_init(&free_db_area->idx_lock, 1);
	63	}
	64
	65	static void __iomem get_db_area(struct hinic_func_to_io func_to_io)
	66	{
	67	struct hinic_free_db_area *free_db_area = &func_to_io->free_db_area;
	68	int pos, idx;
	69
	70	down(&free_db_area->idx_lock);
	71
	72	free_db_area->num_free--;
	73
	74	if (free_db_area->num_free < 0) {
	75	free_db_area->num_free++;
	76	up(&free_db_area->idx_lock);
	77	return ERR_PTR(-ENOMEM);
	78	}
	79
	80	pos = free_db_area->alloc_pos++;
	81	pos &= HINIC_DB_MAX_AREAS - 1;
	82
	83	idx = free_db_area->db_idx[pos];
	84
	85	free_db_area->db_idx[pos] = -1;
	86
	87	up(&free_db_area->idx_lock);
	88
	89	return func_to_io->db_base + idx * HINIC_DB_PAGE_SIZE;
	90	}
	91
	92	static void return_db_area(struct hinic_func_to_io *func_to_io,
	93	void __iomem *db_base)
	94	{
	95	struct hinic_free_db_area *free_db_area = &func_to_io->free_db_area;
	96	int pos, idx = DB_IDX(db_base, func_to_io->db_base);
	97
	98	down(&free_db_area->idx_lock);
	99
	100	pos = free_db_area->return_pos++;
	101	pos &= HINIC_DB_MAX_AREAS - 1;
	102
	103	free_db_area->db_idx[pos] = idx;
	104
	105	free_db_area->num_free++;
	106
	107	up(&free_db_area->idx_lock);
	108	}
	109
53e7d6fe AK	110	static int write_sq_ctxts(struct hinic_func_to_io *func_to_io, u16 base_qpn,
	111	u16 num_sqs)
	112	{
	113	struct hinic_hwif *hwif = func_to_io->hwif;
	114	struct hinic_sq_ctxt_block *sq_ctxt_block;
	115	struct pci_dev *pdev = hwif->pdev;
	116	struct hinic_cmdq_buf cmdq_buf;
	117	struct hinic_sq_ctxt *sq_ctxt;
	118	struct hinic_qp *qp;
	119	u64 out_param;
	120	int err, i;
	121
	122	err = hinic_alloc_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf);
	123	if (err) {
	124	dev_err(&pdev->dev, "Failed to allocate cmdq buf\n");
	125	return err;
	126	}
	127
	128	sq_ctxt_block = cmdq_buf.buf;
	129	sq_ctxt = sq_ctxt_block->sq_ctxt;
	130
	131	hinic_qp_prepare_header(&sq_ctxt_block->hdr, HINIC_QP_CTXT_TYPE_SQ,
	132	num_sqs, func_to_io->max_qps);
	133	for (i = 0; i < num_sqs; i++) {
	134	qp = &func_to_io->qps[i];
	135
	136	hinic_sq_prepare_ctxt(&sq_ctxt[i], &qp->sq,
	137	base_qpn + qp->q_id);
	138	}
	139
	140	cmdq_buf.size = HINIC_SQ_CTXT_SIZE(num_sqs);
	141
	142	err = hinic_cmdq_direct_resp(&func_to_io->cmdqs, HINIC_MOD_L2NIC,
	143	IO_CMD_MODIFY_QUEUE_CTXT, &cmdq_buf,
	144	&out_param);
	145	if ((err) \|\| (out_param != 0)) {
	146	dev_err(&pdev->dev, "Failed to set SQ ctxts\n");
	147	err = -EFAULT;
	148	}
	149
	150	hinic_free_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf);
	151	return err;
	152	}
	153
	154	static int write_rq_ctxts(struct hinic_func_to_io *func_to_io, u16 base_qpn,
	155	u16 num_rqs)
	156	{
	157	struct hinic_hwif *hwif = func_to_io->hwif;
	158	struct hinic_rq_ctxt_block *rq_ctxt_block;
	159	struct pci_dev *pdev = hwif->pdev;
	160	struct hinic_cmdq_buf cmdq_buf;
	161	struct hinic_rq_ctxt *rq_ctxt;
	162	struct hinic_qp *qp;
	163	u64 out_param;
	164	int err, i;
	165
	166	err = hinic_alloc_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf);
	167	if (err) {
	168	dev_err(&pdev->dev, "Failed to allocate cmdq buf\n");
	169	return err;
	170	}
	171
	172	rq_ctxt_block = cmdq_buf.buf;
	173	rq_ctxt = rq_ctxt_block->rq_ctxt;
174
175	hinic_qp_prepare_header(&rq_ctxt_block->hdr, HINIC_QP_CTXT_TYPE_RQ,
176	num_rqs, func_to_io->max_qps);
177	for (i = 0; i < num_rqs; i++) {
178	qp = &func_to_io->qps[i];
179
180	hinic_rq_prepare_ctxt(&rq_ctxt[i], &qp->rq,
181	base_qpn + qp->q_id);
182	}
183
184	cmdq_buf.size = HINIC_RQ_CTXT_SIZE(num_rqs);
185
186	err = hinic_cmdq_direct_resp(&func_to_io->cmdqs, HINIC_MOD_L2NIC,
187	IO_CMD_MODIFY_QUEUE_CTXT, &cmdq_buf,
188	&out_param);
189	if ((err) \|\| (out_param != 0)) {
190	dev_err(&pdev->dev, "Failed to set RQ ctxts\n");
191	err = -EFAULT;
192	}
193
194	hinic_free_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf);
195	return err;
196	}
197
198	/**
199	* write_qp_ctxts - write the qp ctxt to HW
200	* @func_to_io: func to io channel that holds the IO components
201	* @base_qpn: first qp number
202	* @num_qps: number of qps to write
203	*
204	* Return 0 - Success, negative - Failure
205	**/
206	static int write_qp_ctxts(struct hinic_func_to_io *func_to_io, u16 base_qpn,
207	u16 num_qps)
208	{
209	return (write_sq_ctxts(func_to_io, base_qpn, num_qps) \|\|
210	write_rq_ctxts(func_to_io, base_qpn, num_qps));
211	}
212
c3e79baf AK	213	/**
	214	* init_qp - Initialize a Queue Pair
	215	* @func_to_io: func to io channel that holds the IO components
	216	* @qp: pointer to the qp to initialize
	217	* @q_id: the id of the qp
	218	* @sq_msix_entry: msix entry for sq
	219	* @rq_msix_entry: msix entry for rq
	220	*
	221	* Return 0 - Success, negative - Failure
	222	**/
	223	static int init_qp(struct hinic_func_to_io *func_to_io,
	224	struct hinic_qp *qp, int q_id,
	225	struct msix_entry *sq_msix_entry,
	226	struct msix_entry *rq_msix_entry)
	227	{
b15a9f37 AK	228	struct hinic_hwif *hwif = func_to_io->hwif;
b15a9f37 AK	229	struct pci_dev *pdev = hwif->pdev;
f91090f7	230	void __iomem *db_base;
b15a9f37 AK	231	int err;
	232
	233	qp->q_id = q_id;
	234
	235	err = hinic_wq_allocate(&func_to_io->wqs, &func_to_io->sq_wq[q_id],
	236	HINIC_SQ_WQEBB_SIZE, HINIC_SQ_PAGE_SIZE,
	237	HINIC_SQ_DEPTH, HINIC_SQ_WQE_MAX_SIZE);
	238	if (err) {
	239	dev_err(&pdev->dev, "Failed to allocate WQ for SQ\n");
	240	return err;
	241	}
	242
	243	err = hinic_wq_allocate(&func_to_io->wqs, &func_to_io->rq_wq[q_id],
	244	HINIC_RQ_WQEBB_SIZE, HINIC_RQ_PAGE_SIZE,
	245	HINIC_RQ_DEPTH, HINIC_RQ_WQE_SIZE);
	246	if (err) {
	247	dev_err(&pdev->dev, "Failed to allocate WQ for RQ\n");
	248	goto err_rq_alloc;
	249	}
	250
f91090f7 AK	251	db_base = get_db_area(func_to_io);
	252	if (IS_ERR(db_base)) {
	253	dev_err(&pdev->dev, "Failed to get DB area for SQ\n");
	254	err = PTR_ERR(db_base);
	255	goto err_get_db;
	256	}
	257
	258	func_to_io->sq_db[q_id] = db_base;
	259
	260	err = hinic_init_sq(&qp->sq, hwif, &func_to_io->sq_wq[q_id],
	261	sq_msix_entry,
	262	CI_ADDR(func_to_io->ci_addr_base, q_id),
	263	CI_ADDR(func_to_io->ci_dma_base, q_id), db_base);
	264	if (err) {
	265	dev_err(&pdev->dev, "Failed to init SQ\n");
	266	goto err_sq_init;
	267	}
	268
	269	err = hinic_init_rq(&qp->rq, hwif, &func_to_io->rq_wq[q_id],
	270	rq_msix_entry);
	271	if (err) {
	272	dev_err(&pdev->dev, "Failed to init RQ\n");
	273	goto err_rq_init;
	274	}
	275
c3e79baf	276	return 0;
b15a9f37	277
f91090f7 AK	278	err_rq_init:
	279	hinic_clean_sq(&qp->sq);
	280
	281	err_sq_init:
	282	return_db_area(func_to_io, db_base);
	283
	284	err_get_db:
	285	hinic_wq_free(&func_to_io->wqs, &func_to_io->rq_wq[q_id]);
	286
b15a9f37 AK	287	err_rq_alloc:
	288	hinic_wq_free(&func_to_io->wqs, &func_to_io->sq_wq[q_id]);
	289	return err;
c3e79baf AK	290	}
	291
	292	/**
	293	* destroy_qp - Clean the resources of a Queue Pair
	294	* @func_to_io: func to io channel that holds the IO components
	295	* @qp: pointer to the qp to clean
	296	**/
	297	static void destroy_qp(struct hinic_func_to_io *func_to_io,
	298	struct hinic_qp *qp)
	299	{
b15a9f37 AK	300	int q_id = qp->q_id;
b15a9f37 AK	301
f91090f7 AK	302	hinic_clean_rq(&qp->rq);
	303	hinic_clean_sq(&qp->sq);
	304
	305	return_db_area(func_to_io, func_to_io->sq_db[q_id]);
	306
b15a9f37 AK	307	hinic_wq_free(&func_to_io->wqs, &func_to_io->rq_wq[q_id]);
b15a9f37 AK	308	hinic_wq_free(&func_to_io->wqs, &func_to_io->sq_wq[q_id]);
c3e79baf AK	309	}
	310
	311	/**
	312	* hinic_io_create_qps - Create Queue Pairs
	313	* @func_to_io: func to io channel that holds the IO components
	314	* @base_qpn: base qp number
	315	* @num_qps: number queue pairs to create
	316	* @sq_msix_entry: msix entries for sq
	317	* @rq_msix_entry: msix entries for rq
	318	*
	319	* Return 0 - Success, negative - Failure
	320	**/
	321	int hinic_io_create_qps(struct hinic_func_to_io *func_to_io,
	322	u16 base_qpn, int num_qps,
	323	struct msix_entry *sq_msix_entries,
	324	struct msix_entry *rq_msix_entries)
	325	{
	326	struct hinic_hwif *hwif = func_to_io->hwif;
	327	struct pci_dev *pdev = hwif->pdev;
f91090f7 AK	328	size_t qps_size, wq_size, db_size;
f91090f7 AK	329	void *ci_addr_base;
c3e79baf AK	330	int i, j, err;
	331
	332	qps_size = num_qps * sizeof(*func_to_io->qps);
	333	func_to_io->qps = devm_kzalloc(&pdev->dev, qps_size, GFP_KERNEL);
	334	if (!func_to_io->qps)
	335	return -ENOMEM;
	336
b15a9f37 AK	337	wq_size = num_qps * sizeof(*func_to_io->sq_wq);
	338	func_to_io->sq_wq = devm_kzalloc(&pdev->dev, wq_size, GFP_KERNEL);
	339	if (!func_to_io->sq_wq) {
	340	err = -ENOMEM;
	341	goto err_sq_wq;
	342	}
	343
	344	wq_size = num_qps * sizeof(*func_to_io->rq_wq);
	345	func_to_io->rq_wq = devm_kzalloc(&pdev->dev, wq_size, GFP_KERNEL);
	346	if (!func_to_io->rq_wq) {
	347	err = -ENOMEM;
	348	goto err_rq_wq;
	349	}
	350
f91090f7 AK	351	db_size = num_qps * sizeof(*func_to_io->sq_db);
	352	func_to_io->sq_db = devm_kzalloc(&pdev->dev, db_size, GFP_KERNEL);
	353	if (!func_to_io->sq_db) {
	354	err = -ENOMEM;
	355	goto err_sq_db;
	356	}
	357
750afb08 LC	358	ci_addr_base = dma_alloc_coherent(&pdev->dev, CI_TABLE_SIZE(num_qps),
	359	&func_to_io->ci_dma_base,
	360	GFP_KERNEL);
f91090f7 AK	361	if (!ci_addr_base) {
	362	dev_err(&pdev->dev, "Failed to allocate CI area\n");
	363	err = -ENOMEM;
	364	goto err_ci_base;
	365	}
	366
	367	func_to_io->ci_addr_base = ci_addr_base;
	368
c3e79baf AK	369	for (i = 0; i < num_qps; i++) {
	370	err = init_qp(func_to_io, &func_to_io->qps[i], i,
	371	&sq_msix_entries[i], &rq_msix_entries[i]);
	372	if (err) {
	373	dev_err(&pdev->dev, "Failed to create QP %d\n", i);
	374	goto err_init_qp;
	375	}
	376	}
	377
53e7d6fe AK	378	err = write_qp_ctxts(func_to_io, base_qpn, num_qps);
	379	if (err) {
	380	dev_err(&pdev->dev, "Failed to init QP ctxts\n");
	381	goto err_write_qp_ctxts;
	382	}
	383
c3e79baf AK	384	return 0;
c3e79baf AK	385
53e7d6fe	386	err_write_qp_ctxts:
c3e79baf AK	387	err_init_qp:
	388	for (j = 0; j < i; j++)
	389	destroy_qp(func_to_io, &func_to_io->qps[j]);
	390
f91090f7 AK	391	dma_free_coherent(&pdev->dev, CI_TABLE_SIZE(num_qps),
	392	func_to_io->ci_addr_base, func_to_io->ci_dma_base);
	393
	394	err_ci_base:
	395	devm_kfree(&pdev->dev, func_to_io->sq_db);
	396
	397	err_sq_db:
b15a9f37 AK	398	devm_kfree(&pdev->dev, func_to_io->rq_wq);
	399
	400	err_rq_wq:
	401	devm_kfree(&pdev->dev, func_to_io->sq_wq);
	402
	403	err_sq_wq:
c3e79baf AK	404	devm_kfree(&pdev->dev, func_to_io->qps);
	405	return err;
	406	}
	407
	408	/**
	409	* hinic_io_destroy_qps - Destroy the IO Queue Pairs
	410	* @func_to_io: func to io channel that holds the IO components
	411	* @num_qps: number queue pairs to destroy
	412	**/
	413	void hinic_io_destroy_qps(struct hinic_func_to_io *func_to_io, int num_qps)
	414	{
	415	struct hinic_hwif *hwif = func_to_io->hwif;
	416	struct pci_dev *pdev = hwif->pdev;
f91090f7	417	size_t ci_table_size;
c3e79baf AK	418	int i;
c3e79baf AK	419
f91090f7 AK	420	ci_table_size = CI_TABLE_SIZE(num_qps);
f91090f7 AK	421
c3e79baf AK	422	for (i = 0; i < num_qps; i++)
	423	destroy_qp(func_to_io, &func_to_io->qps[i]);
	424
f91090f7 AK	425	dma_free_coherent(&pdev->dev, ci_table_size, func_to_io->ci_addr_base,
	426	func_to_io->ci_dma_base);
	427
	428	devm_kfree(&pdev->dev, func_to_io->sq_db);
	429
b15a9f37 AK	430	devm_kfree(&pdev->dev, func_to_io->rq_wq);
	431	devm_kfree(&pdev->dev, func_to_io->sq_wq);
	432
c3e79baf AK	433	devm_kfree(&pdev->dev, func_to_io->qps);
	434	}
	435
	436	/**
	437	* hinic_io_init - Initialize the IO components
	438	* @func_to_io: func to io channel that holds the IO components
	439	* @hwif: HW interface for accessing IO
	440	* @max_qps: maximum QPs in HW
	441	* @num_ceqs: number completion event queues
	442	* @ceq_msix_entries: msix entries for ceqs
	443	*
	444	* Return 0 - Success, negative - Failure
	445	**/
	446	int hinic_io_init(struct hinic_func_to_io *func_to_io,
	447	struct hinic_hwif *hwif, u16 max_qps, int num_ceqs,
	448	struct msix_entry *ceq_msix_entries)
	449	{
b15a9f37	450	struct pci_dev *pdev = hwif->pdev;
53e7d6fe AK	451	enum hinic_cmdq_type cmdq, type;
53e7d6fe AK	452	void __iomem *db_area;
b15a9f37 AK	453	int err;
b15a9f37 AK	454
c3e79baf AK	455	func_to_io->hwif = hwif;
	456	func_to_io->qps = NULL;
	457	func_to_io->max_qps = max_qps;
	458
fc9319e4 AK	459	err = hinic_ceqs_init(&func_to_io->ceqs, hwif, num_ceqs,
	460	HINIC_DEFAULT_CEQ_LEN, HINIC_EQ_PAGE_SIZE,
	461	ceq_msix_entries);
	462	if (err) {
	463	dev_err(&pdev->dev, "Failed to init CEQs\n");
	464	return err;
	465	}
	466
b15a9f37 AK	467	err = hinic_wqs_alloc(&func_to_io->wqs, 2 * max_qps, hwif);
	468	if (err) {
	469	dev_err(&pdev->dev, "Failed to allocate WQS for IO\n");
fc9319e4	470	goto err_wqs_alloc;
b15a9f37 AK	471	}
b15a9f37 AK	472
f91090f7 AK	473	func_to_io->db_base = pci_ioremap_bar(pdev, HINIC_PCI_DB_BAR);
	474	if (!func_to_io->db_base) {
	475	dev_err(&pdev->dev, "Failed to remap IO DB area\n");
	476	err = -ENOMEM;
	477	goto err_db_ioremap;
	478	}
	479
	480	init_db_area_idx(&func_to_io->free_db_area);
53e7d6fe AK	481
	482	for (cmdq = HINIC_CMDQ_SYNC; cmdq < HINIC_MAX_CMDQ_TYPES; cmdq++) {
	483	db_area = get_db_area(func_to_io);
	484	if (IS_ERR(db_area)) {
	485	dev_err(&pdev->dev, "Failed to get cmdq db area\n");
	486	err = PTR_ERR(db_area);
	487	goto err_db_area;
	488	}
	489
	490	func_to_io->cmdq_db_area[cmdq] = db_area;
	491	}
	492
	493	err = hinic_init_cmdqs(&func_to_io->cmdqs, hwif,
	494	func_to_io->cmdq_db_area);
	495	if (err) {
	496	dev_err(&pdev->dev, "Failed to initialize cmdqs\n");
	497	goto err_init_cmdqs;
	498	}
	499
c3e79baf	500	return 0;
f91090f7	501
53e7d6fe AK	502	err_init_cmdqs:
	503	err_db_area:
	504	for (type = HINIC_CMDQ_SYNC; type < cmdq; type++)
	505	return_db_area(func_to_io, func_to_io->cmdq_db_area[type]);
	506
	507	iounmap(func_to_io->db_base);
	508
f91090f7 AK	509	err_db_ioremap:
f91090f7 AK	510	hinic_wqs_free(&func_to_io->wqs);
fc9319e4 AK	511
	512	err_wqs_alloc:
	513	hinic_ceqs_free(&func_to_io->ceqs);
f91090f7	514	return err;
c3e79baf AK	515	}
	516
	517	/**
	518	* hinic_io_free - Free the IO components
	519	* @func_to_io: func to io channel that holds the IO components
	520	**/
	521	void hinic_io_free(struct hinic_func_to_io *func_to_io)
	522	{
53e7d6fe AK	523	enum hinic_cmdq_type cmdq;
	524
	525	hinic_free_cmdqs(&func_to_io->cmdqs);
	526
	527	for (cmdq = HINIC_CMDQ_SYNC; cmdq < HINIC_MAX_CMDQ_TYPES; cmdq++)
	528	return_db_area(func_to_io, func_to_io->cmdq_db_area[cmdq]);
	529
f91090f7	530	iounmap(func_to_io->db_base);
b15a9f37	531	hinic_wqs_free(&func_to_io->wqs);
fc9319e4	532	hinic_ceqs_free(&func_to_io->ceqs);
c3e79baf	533	}