[mirror_ubuntu-jammy-kernel.git] / drivers / crypto / cavium / nitrox / nitrox_main.c

#include <linux/aer.h>
#include <linux/delay.h>
#include <linux/debugfs.h>
#include <linux/firmware.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/pci.h>
#include <linux/pci_ids.h>

#include "nitrox_dev.h"
#include "nitrox_common.h"
#include "nitrox_csr.h"
#include "nitrox_hal.h"

#define CNN55XX_DEV_ID	0x12
#define MAX_PF_QUEUES	64
#define UCODE_HLEN 48
#define SE_GROUP 0

#define DRIVER_VERSION "1.1"
#define FW_DIR "cavium/"
/* SE microcode */
#define SE_FW	FW_DIR "cnn55xx_se.fw"

static const char nitrox_driver_name[] = "CNN55XX";

static LIST_HEAD(ndevlist);
static DEFINE_MUTEX(devlist_lock);
static unsigned int num_devices;

/**
 * nitrox_pci_tbl - PCI Device ID Table
 */
static const struct pci_device_id nitrox_pci_tbl[] = {
	{PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
	/* required last entry */
	{0, }
};
MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);

static unsigned int qlen = DEFAULT_CMD_QLEN;
module_param(qlen, uint, 0644);
MODULE_PARM_DESC(qlen, "Command queue length - default 2048");

#ifdef CONFIG_PCI_IOV
int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs);
#else
int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs)
{
	return 0;
}
#endif

/**
 * struct ucode - Firmware Header
 * @id: microcode ID
 * @version: firmware version
 * @code_size: code section size
 * @raz: alignment
 * @code: code section
 */
struct ucode {
	u8 id;
	char version[VERSION_LEN - 1];
	__be32 code_size;
	u8 raz[12];
	u64 code[0];
};

/**
 * write_to_ucd_unit - Write Firmware to NITROX UCD unit
 */
static void write_to_ucd_unit(struct nitrox_device *ndev,
			      struct ucode *ucode)
{
	u32 code_size = be32_to_cpu(ucode->code_size) * 2;
	u64 offset, data;
	int i = 0;

	/*
	 * UCD structure
	 *
	 *  -------------
	 *  |    BLK 7  |
	 *  -------------
	 *  |    BLK 6  |
	 *  -------------
	 *  |    ...    |
	 *  -------------
	 *  |    BLK 0  |
	 *  -------------
	 *  Total of 8 blocks, each size 32KB
	 */

	/* set the block number */
	offset = UCD_UCODE_LOAD_BLOCK_NUM;
	nitrox_write_csr(ndev, offset, 0);

	code_size = roundup(code_size, 8);
	while (code_size) {
		data = ucode->code[i];
		/* write 8 bytes at a time */
		offset = UCD_UCODE_LOAD_IDX_DATAX(i);
		nitrox_write_csr(ndev, offset, data);
		code_size -= 8;
		i++;
	}

	/* put all SE cores in group 0 */
	offset = POM_GRP_EXECMASKX(SE_GROUP);
	nitrox_write_csr(ndev, offset, (~0ULL));

	for (i = 0; i < ndev->hw.se_cores; i++) {
		/*
		 * write block number and firware length
		 * bit:<2:0> block number
		 * bit:3 is set SE uses 32KB microcode
		 * bit:3 is clear SE uses 64KB microcode
		 */
		offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
		nitrox_write_csr(ndev, offset, 0x8);
	}
	usleep_range(300, 400);
}

static int nitrox_load_fw(struct nitrox_device *ndev, const char *fw_name)
{
	const struct firmware *fw;
	struct ucode *ucode;
	int ret;

	dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);

	ret = request_firmware(&fw, fw_name, DEV(ndev));
	if (ret < 0) {
		dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
		return ret;
	}

	ucode = (struct ucode *)fw->data;
	/* copy the firmware version */
	memcpy(ndev->hw.fw_name, ucode->version, (VERSION_LEN - 2));
	ndev->hw.fw_name[VERSION_LEN - 1] = '\0';

	write_to_ucd_unit(ndev, ucode);
	release_firmware(fw);

	return 0;
}

/**
 * nitrox_add_to_devlist - add NITROX device to global device list
 * @ndev: NITROX device
 */
static int nitrox_add_to_devlist(struct nitrox_device *ndev)
{
	struct nitrox_device *dev;
	int ret = 0;

	INIT_LIST_HEAD(&ndev->list);
	refcount_set(&ndev->refcnt, 1);

	mutex_lock(&devlist_lock);
	list_for_each_entry(dev, &ndevlist, list) {
		if (dev == ndev) {
			ret = -EEXIST;
			goto unlock;
		}
	}
	ndev->idx = num_devices++;
	list_add_tail(&ndev->list, &ndevlist);
unlock:
	mutex_unlock(&devlist_lock);
	return ret;
}

/**
 * nitrox_remove_from_devlist - remove NITROX device from
 *   global device list
 * @ndev: NITROX device
 */
static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
{
	mutex_lock(&devlist_lock);
	list_del(&ndev->list);
	num_devices--;
	mutex_unlock(&devlist_lock);
}

struct nitrox_device *nitrox_get_first_device(void)
{
	struct nitrox_device *ndev = NULL;

	mutex_lock(&devlist_lock);
	list_for_each_entry(ndev, &ndevlist, list) {
		if (nitrox_ready(ndev))
			break;
	}
	mutex_unlock(&devlist_lock);
	if (!ndev)
		return NULL;

	refcount_inc(&ndev->refcnt);
	/* barrier to sync with other cpus */
	smp_mb__after_atomic();
	return ndev;
}

void nitrox_put_device(struct nitrox_device *ndev)
{
	if (!ndev)
		return;

	refcount_dec(&ndev->refcnt);
	/* barrier to sync with other cpus */
	smp_mb__after_atomic();
}

static int nitrox_reset_device(struct pci_dev *pdev)
{
	int pos = 0;

	pos = pci_save_state(pdev);
	if (pos) {
		dev_err(&pdev->dev, "Failed to save pci state\n");
		return -ENOMEM;
	}

	pos = pci_pcie_cap(pdev);
	if (!pos)
		return -ENOTTY;

	if (!pci_wait_for_pending_transaction(pdev))
		dev_err(&pdev->dev, "waiting for pending transaction\n");

	pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
	msleep(100);
	pci_restore_state(pdev);

	return 0;
}

static int nitrox_pf_sw_init(struct nitrox_device *ndev)
{
	int err;

	err = nitrox_common_sw_init(ndev);
	if (err)
		return err;

	err = nitrox_pf_init_isr(ndev);
	if (err)
		nitrox_common_sw_cleanup(ndev);

	return err;
}

static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
{
	nitrox_pf_cleanup_isr(ndev);
	nitrox_common_sw_cleanup(ndev);
}

/**
 * nitrox_bist_check - Check NITORX BIST registers status
 * @ndev: NITROX device
 */
static int nitrox_bist_check(struct nitrox_device *ndev)
{
	u64 value = 0;
	int i;

	for (i = 0; i < NR_CLUSTERS; i++) {
		value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
		value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
	}
	value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
	value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
	value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
	value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
	value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
	value += nitrox_read_csr(ndev, POM_BIST_REG);
	value += nitrox_read_csr(ndev, BMI_BIST_REG);
	value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
	value += nitrox_read_csr(ndev, BMO_BIST_REG);
	value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
	value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
	if (value)
		return -EIO;
	return 0;
}

static int nitrox_pf_hw_init(struct nitrox_device *ndev)
{
	int err;

	err = nitrox_bist_check(ndev);
	if (err) {
		dev_err(&ndev->pdev->dev, "BIST check failed\n");
		return err;
	}
	/* get cores information */
	nitrox_get_hwinfo(ndev);

	nitrox_config_nps_unit(ndev);
	nitrox_config_pom_unit(ndev);
	nitrox_config_efl_unit(ndev);
	/* configure IO units */
	nitrox_config_bmi_unit(ndev);
	nitrox_config_bmo_unit(ndev);
	/* configure Local Buffer Cache */
	nitrox_config_lbc_unit(ndev);
	nitrox_config_rand_unit(ndev);

	/* load firmware on SE cores */
	err = nitrox_load_fw(ndev, SE_FW);
	if (err)
		return err;

	nitrox_config_emu_unit(ndev);

	return 0;
}

#if IS_ENABLED(CONFIG_DEBUG_FS)
static int registers_show(struct seq_file *s, void *v)
{
	struct nitrox_device *ndev = s->private;
	u64 offset;

	/* NPS DMA stats */
	offset = NPS_STATS_PKT_DMA_RD_CNT;
	seq_printf(s, "NPS_STATS_PKT_DMA_RD_CNT  0x%016llx\n",
		   nitrox_read_csr(ndev, offset));
	offset = NPS_STATS_PKT_DMA_WR_CNT;
	seq_printf(s, "NPS_STATS_PKT_DMA_WR_CNT  0x%016llx\n",
		   nitrox_read_csr(ndev, offset));

	/* BMI/BMO stats */
	offset = BMI_NPS_PKT_CNT;
	seq_printf(s, "BMI_NPS_PKT_CNT  0x%016llx\n",
		   nitrox_read_csr(ndev, offset));
	offset = BMO_NPS_SLC_PKT_CNT;
	seq_printf(s, "BMO_NPS_PKT_CNT  0x%016llx\n",
		   nitrox_read_csr(ndev, offset));

	return 0;
}

static int registers_open(struct inode *inode, struct file *file)
{
	return single_open(file, registers_show, inode->i_private);
}

static const struct file_operations register_fops = {
	.owner = THIS_MODULE,
	.open = registers_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = single_release,
};

static int firmware_show(struct seq_file *s, void *v)
{
	struct nitrox_device *ndev = s->private;

	seq_printf(s, "Version: %s\n", ndev->hw.fw_name);
	return 0;
}

static int firmware_open(struct inode *inode, struct file *file)
{
	return single_open(file, firmware_show, inode->i_private);
}

static const struct file_operations firmware_fops = {
	.owner = THIS_MODULE,
	.open = firmware_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = single_release,
};

static int nitrox_show(struct seq_file *s, void *v)
{
	struct nitrox_device *ndev = s->private;

	seq_printf(s, "NITROX-5 [idx: %d]\n", ndev->idx);
	seq_printf(s, "  Revision ID: 0x%0x\n", ndev->hw.revision_id);
	seq_printf(s, "  Cores [AE: %u  SE: %u]\n",
		   ndev->hw.ae_cores, ndev->hw.se_cores);
	seq_printf(s, "  Number of Queues: %u\n", ndev->nr_queues);
	seq_printf(s, "  Queue length: %u\n", ndev->qlen);
	seq_printf(s, "  Node: %u\n", ndev->node);

	return 0;
}

static int nitrox_open(struct inode *inode, struct file *file)
{
	return single_open(file, nitrox_show, inode->i_private);
}

static const struct file_operations nitrox_fops = {
	.owner = THIS_MODULE,
	.open = nitrox_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = single_release,
};

static void nitrox_debugfs_exit(struct nitrox_device *ndev)
{
	debugfs_remove_recursive(ndev->debugfs_dir);
	ndev->debugfs_dir = NULL;
}

static int nitrox_debugfs_init(struct nitrox_device *ndev)
{
	struct dentry *dir, *f;

	dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
	if (!dir)
		return -ENOMEM;

	ndev->debugfs_dir = dir;
	f = debugfs_create_file("counters", 0400, dir, ndev, &register_fops);
	if (!f)
		goto err;
	f = debugfs_create_file("firmware", 0400, dir, ndev, &firmware_fops);
	if (!f)
		goto err;
	f = debugfs_create_file("nitrox", 0400, dir, ndev, &nitrox_fops);
	if (!f)
		goto err;

	return 0;

err:
	nitrox_debugfs_exit(ndev);
	return -ENODEV;
}
#else
static int nitrox_debugfs_init(struct nitrox_device *ndev)
{
	return 0;
}

static void nitrox_debugfs_exit(struct nitrox_device *ndev)
{
}
#endif

/**
 * nitrox_probe - NITROX Initialization function.
 * @pdev: PCI device information struct
 * @id: entry in nitrox_pci_tbl
 *
 * Return: 0, if the driver is bound to the device, or
 *         a negative error if there is failure.
 */
static int nitrox_probe(struct pci_dev *pdev,
			const struct pci_device_id *id)
{
	struct nitrox_device *ndev;
	int err;

	dev_info_once(&pdev->dev, "%s driver version %s\n",
		      nitrox_driver_name, DRIVER_VERSION);

	err = pci_enable_device_mem(pdev);
	if (err)
		return err;

	/* do FLR */
	err = nitrox_reset_device(pdev);
	if (err) {
		dev_err(&pdev->dev, "FLR failed\n");
		pci_disable_device(pdev);
		return err;
	}

	if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
		dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
	} else {
		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
		if (err) {
			dev_err(&pdev->dev, "DMA configuration failed\n");
			pci_disable_device(pdev);
			return err;
		}
	}

	err = pci_request_mem_regions(pdev, nitrox_driver_name);
	if (err) {
		pci_disable_device(pdev);
		return err;
	}
	pci_set_master(pdev);

	ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
	if (!ndev) {
		err = -ENOMEM;
		goto ndev_fail;
	}

	pci_set_drvdata(pdev, ndev);
	ndev->pdev = pdev;

	/* add to device list */
	nitrox_add_to_devlist(ndev);

	ndev->hw.vendor_id = pdev->vendor;
	ndev->hw.device_id = pdev->device;
	ndev->hw.revision_id = pdev->revision;
	/* command timeout in jiffies */
	ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
	ndev->node = dev_to_node(&pdev->dev);
	if (ndev->node == NUMA_NO_NODE)
		ndev->node = 0;

	ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
				 pci_resource_len(pdev, 0));
	if (!ndev->bar_addr) {
		err = -EIO;
		goto ioremap_err;
	}
	/* allocate command queus based on cpus, max queues are 64 */
	ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
	ndev->qlen = qlen;

	err = nitrox_pf_sw_init(ndev);
	if (err)
		goto ioremap_err;

	err = nitrox_pf_hw_init(ndev);
	if (err)
		goto pf_hw_fail;

	err = nitrox_debugfs_init(ndev);
	if (err)
		goto pf_hw_fail;

	atomic_set(&ndev->state, __NDEV_READY);
	/* barrier to sync with other cpus */
	smp_mb__after_atomic();

	err = nitrox_crypto_register();
	if (err)
		goto crypto_fail;

	return 0;

crypto_fail:
	nitrox_debugfs_exit(ndev);
	atomic_set(&ndev->state, __NDEV_NOT_READY);
	/* barrier to sync with other cpus */
	smp_mb__after_atomic();
pf_hw_fail:
	nitrox_pf_sw_cleanup(ndev);
ioremap_err:
	nitrox_remove_from_devlist(ndev);
	kfree(ndev);
	pci_set_drvdata(pdev, NULL);
ndev_fail:
	pci_release_mem_regions(pdev);
	pci_disable_device(pdev);
	return err;
}

/**
 * nitrox_remove - Unbind the driver from the device.
 * @pdev: PCI device information struct
 */
static void nitrox_remove(struct pci_dev *pdev)
{
	struct nitrox_device *ndev = pci_get_drvdata(pdev);

	if (!ndev)
		return;

	if (!refcount_dec_and_test(&ndev->refcnt)) {
		dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
			refcount_read(&ndev->refcnt));
		return;
	}

	dev_info(DEV(ndev), "Removing Device %x:%x\n",
		 ndev->hw.vendor_id, ndev->hw.device_id);

	atomic_set(&ndev->state, __NDEV_NOT_READY);
	/* barrier to sync with other cpus */
	smp_mb__after_atomic();

	nitrox_remove_from_devlist(ndev);

#ifdef CONFIG_PCI_IOV
	/* disable SR-IOV */
	nitrox_sriov_configure(pdev, 0);
#endif
	nitrox_crypto_unregister();
	nitrox_debugfs_exit(ndev);
	nitrox_pf_sw_cleanup(ndev);

	iounmap(ndev->bar_addr);
	kfree(ndev);

	pci_set_drvdata(pdev, NULL);
	pci_release_mem_regions(pdev);
	pci_disable_device(pdev);
}

static void nitrox_shutdown(struct pci_dev *pdev)
{
	pci_set_drvdata(pdev, NULL);
	pci_release_mem_regions(pdev);
	pci_disable_device(pdev);
}

static struct pci_driver nitrox_driver = {
	.name = nitrox_driver_name,
	.id_table = nitrox_pci_tbl,
	.probe = nitrox_probe,
	.remove	= nitrox_remove,
	.shutdown = nitrox_shutdown,
#ifdef CONFIG_PCI_IOV
	.sriov_configure = nitrox_sriov_configure,
#endif
};

module_pci_driver(nitrox_driver);

MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
MODULE_LICENSE("GPL");
MODULE_VERSION(DRIVER_VERSION);
MODULE_FIRMWARE(SE_FW);
Commit	Line	Data
14fa93cd SJ	1	#include <linux/aer.h>
14fa93cd SJ	2	#include <linux/delay.h>
086eac9e	3	#include <linux/debugfs.h>
14fa93cd SJ	4	#include <linux/firmware.h>
	5	#include <linux/list.h>
	6	#include <linux/module.h>
	7	#include <linux/mutex.h>
	8	#include <linux/pci.h>
	9	#include <linux/pci_ids.h>
	10
	11	#include "nitrox_dev.h"
	12	#include "nitrox_common.h"
	13	#include "nitrox_csr.h"
41a9aca6	14	#include "nitrox_hal.h"
14fa93cd SJ	15
	16	#define CNN55XX_DEV_ID 0x12
	17	#define MAX_PF_QUEUES 64
	18	#define UCODE_HLEN 48
	19	#define SE_GROUP 0
	20
41a9aca6	21	#define DRIVER_VERSION "1.1"
440bf347	22	#define FW_DIR "cavium/"
14fa93cd	23	/* SE microcode */
440bf347	24	#define SE_FW FW_DIR "cnn55xx_se.fw"
14fa93cd SJ	25
	26	static const char nitrox_driver_name[] = "CNN55XX";
	27
	28	static LIST_HEAD(ndevlist);
	29	static DEFINE_MUTEX(devlist_lock);
	30	static unsigned int num_devices;
	31
	32	/**
	33	* nitrox_pci_tbl - PCI Device ID Table
	34	*/
	35	static const struct pci_device_id nitrox_pci_tbl[] = {
	36	{PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
	37	/* required last entry */
	38	{0, }
	39	};
	40	MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);
	41
	42	static unsigned int qlen = DEFAULT_CMD_QLEN;
	43	module_param(qlen, uint, 0644);
	44	MODULE_PARM_DESC(qlen, "Command queue length - default 2048");
	45
41a9aca6 SJ	46	#ifdef CONFIG_PCI_IOV
	47	int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs);
	48	#else
	49	int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs)
	50	{
	51	return 0;
	52	}
	53	#endif
	54
14fa93cd SJ	55	/**
	56	* struct ucode - Firmware Header
	57	* @id: microcode ID
	58	* @version: firmware version
	59	* @code_size: code section size
	60	* @raz: alignment
	61	* @code: code section
	62	*/
	63	struct ucode {
	64	u8 id;
	65	char version[VERSION_LEN - 1];
	66	__be32 code_size;
	67	u8 raz[12];
	68	u64 code[0];
	69	};
	70
	71	/**
	72	* write_to_ucd_unit - Write Firmware to NITROX UCD unit
	73	*/
	74	static void write_to_ucd_unit(struct nitrox_device *ndev,
	75	struct ucode *ucode)
	76	{
	77	u32 code_size = be32_to_cpu(ucode->code_size) * 2;
	78	u64 offset, data;
	79	int i = 0;
	80
	81	/*
	82	* UCD structure
	83	*
	84	* -------------
	85	* \| BLK 7 \|
	86	* -------------
	87	* \| BLK 6 \|
	88	* -------------
	89	* \| ... \|
	90	* -------------
	91	* \| BLK 0 \|
	92	* -------------
	93	* Total of 8 blocks, each size 32KB
	94	*/
	95
	96	/* set the block number */
	97	offset = UCD_UCODE_LOAD_BLOCK_NUM;
	98	nitrox_write_csr(ndev, offset, 0);
	99
	100	code_size = roundup(code_size, 8);
	101	while (code_size) {
	102	data = ucode->code[i];
	103	/* write 8 bytes at a time */
	104	offset = UCD_UCODE_LOAD_IDX_DATAX(i);
	105	nitrox_write_csr(ndev, offset, data);
	106	code_size -= 8;
	107	i++;
	108	}
	109
	110	/* put all SE cores in group 0 */
	111	offset = POM_GRP_EXECMASKX(SE_GROUP);
	112	nitrox_write_csr(ndev, offset, (~0ULL));
	113
	114	for (i = 0; i < ndev->hw.se_cores; i++) {
	115	/*
	116	* write block number and firware length
	117	* bit:<2:0> block number
	118	* bit:3 is set SE uses 32KB microcode
119	* bit:3 is clear SE uses 64KB microcode
120	*/
121	offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
122	nitrox_write_csr(ndev, offset, 0x8);
123	}
124	usleep_range(300, 400);
125	}
126
127	static int nitrox_load_fw(struct nitrox_device ndev, const char fw_name)
128	{
129	const struct firmware *fw;
130	struct ucode *ucode;
131	int ret;
132
133	dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
134
135	ret = request_firmware(&fw, fw_name, DEV(ndev));
136	if (ret < 0) {
137	dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
138	return ret;
139	}
140
141	ucode = (struct ucode *)fw->data;
142	/* copy the firmware version */
143	memcpy(ndev->hw.fw_name, ucode->version, (VERSION_LEN - 2));
144	ndev->hw.fw_name[VERSION_LEN - 1] = '\0';
145
146	write_to_ucd_unit(ndev, ucode);
147	release_firmware(fw);
148
14fa93cd SJ	149	return 0;
	150	}
	151
	152	/**
	153	* nitrox_add_to_devlist - add NITROX device to global device list
	154	* @ndev: NITROX device
	155	*/
	156	static int nitrox_add_to_devlist(struct nitrox_device *ndev)
	157	{
	158	struct nitrox_device *dev;
	159	int ret = 0;
	160
	161	INIT_LIST_HEAD(&ndev->list);
	162	refcount_set(&ndev->refcnt, 1);
	163
	164	mutex_lock(&devlist_lock);
	165	list_for_each_entry(dev, &ndevlist, list) {
	166	if (dev == ndev) {
	167	ret = -EEXIST;
	168	goto unlock;
	169	}
	170	}
	171	ndev->idx = num_devices++;
	172	list_add_tail(&ndev->list, &ndevlist);
	173	unlock:
	174	mutex_unlock(&devlist_lock);
	175	return ret;
	176	}
	177
	178	/**
	179	* nitrox_remove_from_devlist - remove NITROX device from
	180	* global device list
	181	* @ndev: NITROX device
	182	*/
	183	static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
	184	{
	185	mutex_lock(&devlist_lock);
	186	list_del(&ndev->list);
	187	num_devices--;
	188	mutex_unlock(&devlist_lock);
	189	}
	190
f2663872 SJ	191	struct nitrox_device *nitrox_get_first_device(void)
	192	{
	193	struct nitrox_device *ndev = NULL;
	194
	195	mutex_lock(&devlist_lock);
	196	list_for_each_entry(ndev, &ndevlist, list) {
	197	if (nitrox_ready(ndev))
	198	break;
	199	}
	200	mutex_unlock(&devlist_lock);
	201	if (!ndev)
	202	return NULL;
	203
	204	refcount_inc(&ndev->refcnt);
	205	/* barrier to sync with other cpus */
	206	smp_mb__after_atomic();
	207	return ndev;
	208	}
	209
	210	void nitrox_put_device(struct nitrox_device *ndev)
	211	{
	212	if (!ndev)
	213	return;
	214
	215	refcount_dec(&ndev->refcnt);
	216	/* barrier to sync with other cpus */
	217	smp_mb__after_atomic();
	218	}
	219
14fa93cd SJ	220	static int nitrox_reset_device(struct pci_dev *pdev)
	221	{
	222	int pos = 0;
	223
	224	pos = pci_save_state(pdev);
	225	if (pos) {
	226	dev_err(&pdev->dev, "Failed to save pci state\n");
	227	return -ENOMEM;
	228	}
	229
	230	pos = pci_pcie_cap(pdev);
	231	if (!pos)
	232	return -ENOTTY;
	233
	234	if (!pci_wait_for_pending_transaction(pdev))
	235	dev_err(&pdev->dev, "waiting for pending transaction\n");
	236
	237	pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
	238	msleep(100);
	239	pci_restore_state(pdev);
	240
	241	return 0;
	242	}
	243
	244	static int nitrox_pf_sw_init(struct nitrox_device *ndev)
	245	{
	246	int err;
	247
	248	err = nitrox_common_sw_init(ndev);
	249	if (err)
	250	return err;
	251
	252	err = nitrox_pf_init_isr(ndev);
	253	if (err)
	254	nitrox_common_sw_cleanup(ndev);
	255
	256	return err;
	257	}
	258
	259	static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
	260	{
	261	nitrox_pf_cleanup_isr(ndev);
	262	nitrox_common_sw_cleanup(ndev);
	263	}
	264
	265	/**
	266	* nitrox_bist_check - Check NITORX BIST registers status
	267	* @ndev: NITROX device
	268	*/
	269	static int nitrox_bist_check(struct nitrox_device *ndev)
	270	{
	271	u64 value = 0;
	272	int i;
	273
	274	for (i = 0; i < NR_CLUSTERS; i++) {
	275	value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
	276	value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
	277	}
	278	value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
	279	value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
	280	value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
	281	value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
	282	value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
	283	value += nitrox_read_csr(ndev, POM_BIST_REG);
284	value += nitrox_read_csr(ndev, BMI_BIST_REG);
285	value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
286	value += nitrox_read_csr(ndev, BMO_BIST_REG);
287	value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
288	value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
289	if (value)
290	return -EIO;
291	return 0;
292	}
293
14fa93cd SJ	294	static int nitrox_pf_hw_init(struct nitrox_device *ndev)
	295	{
	296	int err;
	297
	298	err = nitrox_bist_check(ndev);
	299	if (err) {
	300	dev_err(&ndev->pdev->dev, "BIST check failed\n");
	301	return err;
	302	}
	303	/* get cores information */
	304	nitrox_get_hwinfo(ndev);
	305
	306	nitrox_config_nps_unit(ndev);
	307	nitrox_config_pom_unit(ndev);
	308	nitrox_config_efl_unit(ndev);
	309	/* configure IO units */
	310	nitrox_config_bmi_unit(ndev);
	311	nitrox_config_bmo_unit(ndev);
	312	/* configure Local Buffer Cache */
	313	nitrox_config_lbc_unit(ndev);
	314	nitrox_config_rand_unit(ndev);
	315
	316	/* load firmware on SE cores */
	317	err = nitrox_load_fw(ndev, SE_FW);
	318	if (err)
	319	return err;
	320
	321	nitrox_config_emu_unit(ndev);
	322
	323	return 0;
	324	}
	325
086eac9e SJ	326	#if IS_ENABLED(CONFIG_DEBUG_FS)
	327	static int registers_show(struct seq_file s, void v)
	328	{
	329	struct nitrox_device *ndev = s->private;
	330	u64 offset;
	331
	332	/* NPS DMA stats */
	333	offset = NPS_STATS_PKT_DMA_RD_CNT;
	334	seq_printf(s, "NPS_STATS_PKT_DMA_RD_CNT 0x%016llx\n",
	335	nitrox_read_csr(ndev, offset));
	336	offset = NPS_STATS_PKT_DMA_WR_CNT;
	337	seq_printf(s, "NPS_STATS_PKT_DMA_WR_CNT 0x%016llx\n",
	338	nitrox_read_csr(ndev, offset));
	339
	340	/* BMI/BMO stats */
	341	offset = BMI_NPS_PKT_CNT;
	342	seq_printf(s, "BMI_NPS_PKT_CNT 0x%016llx\n",
	343	nitrox_read_csr(ndev, offset));
	344	offset = BMO_NPS_SLC_PKT_CNT;
	345	seq_printf(s, "BMO_NPS_PKT_CNT 0x%016llx\n",
	346	nitrox_read_csr(ndev, offset));
	347
	348	return 0;
	349	}
	350
	351	static int registers_open(struct inode inode, struct file file)
	352	{
	353	return single_open(file, registers_show, inode->i_private);
	354	}
	355
	356	static const struct file_operations register_fops = {
	357	.owner = THIS_MODULE,
	358	.open = registers_open,
	359	.read = seq_read,
	360	.llseek = seq_lseek,
	361	.release = single_release,
	362	};
	363
	364	static int firmware_show(struct seq_file s, void v)
	365	{
	366	struct nitrox_device *ndev = s->private;
	367
	368	seq_printf(s, "Version: %s\n", ndev->hw.fw_name);
	369	return 0;
	370	}
	371
	372	static int firmware_open(struct inode inode, struct file file)
	373	{
	374	return single_open(file, firmware_show, inode->i_private);
	375	}
	376
	377	static const struct file_operations firmware_fops = {
	378	.owner = THIS_MODULE,
	379	.open = firmware_open,
	380	.read = seq_read,
	381	.llseek = seq_lseek,
	382	.release = single_release,
	383	};
	384
	385	static int nitrox_show(struct seq_file s, void v)
	386	{
	387	struct nitrox_device *ndev = s->private;
	388
	389	seq_printf(s, "NITROX-5 [idx: %d]\n", ndev->idx);
7c742df5	390	seq_printf(s, " Revision ID: 0x%0x\n", ndev->hw.revision_id);
086eac9e SJ	391	seq_printf(s, " Cores [AE: %u SE: %u]\n",
	392	ndev->hw.ae_cores, ndev->hw.se_cores);
	393	seq_printf(s, " Number of Queues: %u\n", ndev->nr_queues);
	394	seq_printf(s, " Queue length: %u\n", ndev->qlen);
	395	seq_printf(s, " Node: %u\n", ndev->node);
	396
	397	return 0;
	398	}
	399
	400	static int nitrox_open(struct inode inode, struct file file)
	401	{
	402	return single_open(file, nitrox_show, inode->i_private);
	403	}
	404
	405	static const struct file_operations nitrox_fops = {
	406	.owner = THIS_MODULE,
	407	.open = nitrox_open,
	408	.read = seq_read,
	409	.llseek = seq_lseek,
	410	.release = single_release,
	411	};
	412
	413	static void nitrox_debugfs_exit(struct nitrox_device *ndev)
	414	{
	415	debugfs_remove_recursive(ndev->debugfs_dir);
	416	ndev->debugfs_dir = NULL;
	417	}
	418
	419	static int nitrox_debugfs_init(struct nitrox_device *ndev)
	420	{
	421	struct dentry dir, f;
	422
	423	dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
	424	if (!dir)
	425	return -ENOMEM;
	426
	427	ndev->debugfs_dir = dir;
	428	f = debugfs_create_file("counters", 0400, dir, ndev, &register_fops);
	429	if (!f)
	430	goto err;
	431	f = debugfs_create_file("firmware", 0400, dir, ndev, &firmware_fops);
	432	if (!f)
	433	goto err;
	434	f = debugfs_create_file("nitrox", 0400, dir, ndev, &nitrox_fops);
	435	if (!f)
	436	goto err;
	437
	438	return 0;
	439
	440	err:
	441	nitrox_debugfs_exit(ndev);
	442	return -ENODEV;
	443	}
	444	#else
	445	static int nitrox_debugfs_init(struct nitrox_device *ndev)
	446	{
	447	return 0;
	448	}
	449
	450	static void nitrox_debugfs_exit(struct nitrox_device *ndev)
	451	{
	452	}
	453	#endif
	454
14fa93cd SJ	455	/**
	456	* nitrox_probe - NITROX Initialization function.
	457	* @pdev: PCI device information struct
	458	* @id: entry in nitrox_pci_tbl
	459	*
	460	* Return: 0, if the driver is bound to the device, or
	461	* a negative error if there is failure.
	462	*/
	463	static int nitrox_probe(struct pci_dev *pdev,
	464	const struct pci_device_id *id)
	465	{
	466	struct nitrox_device *ndev;
	467	int err;
	468
	469	dev_info_once(&pdev->dev, "%s driver version %s\n",
	470	nitrox_driver_name, DRIVER_VERSION);
	471
	472	err = pci_enable_device_mem(pdev);
	473	if (err)
	474	return err;
	475
	476	/* do FLR */
	477	err = nitrox_reset_device(pdev);
	478	if (err) {
	479	dev_err(&pdev->dev, "FLR failed\n");
	480	pci_disable_device(pdev);
	481	return err;
	482	}
	483
	484	if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
	485	dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
	486	} else {
	487	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
	488	if (err) {
	489	dev_err(&pdev->dev, "DMA configuration failed\n");
	490	pci_disable_device(pdev);
	491	return err;
	492	}
	493	}
	494
	495	err = pci_request_mem_regions(pdev, nitrox_driver_name);
	496	if (err) {
	497	pci_disable_device(pdev);
	498	return err;
	499	}
	500	pci_set_master(pdev);
	501
	502	ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
32e67b9a CJ	503	if (!ndev) {
32e67b9a CJ	504	err = -ENOMEM;
14fa93cd	505	goto ndev_fail;
32e67b9a	506	}
14fa93cd SJ	507
	508	pci_set_drvdata(pdev, ndev);
	509	ndev->pdev = pdev;
	510
	511	/* add to device list */
	512	nitrox_add_to_devlist(ndev);
	513
	514	ndev->hw.vendor_id = pdev->vendor;
	515	ndev->hw.device_id = pdev->device;
	516	ndev->hw.revision_id = pdev->revision;
	517	/* command timeout in jiffies */
	518	ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
	519	ndev->node = dev_to_node(&pdev->dev);
	520	if (ndev->node == NUMA_NO_NODE)
	521	ndev->node = 0;
	522
	523	ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
	524	pci_resource_len(pdev, 0));
	525	if (!ndev->bar_addr) {
	526	err = -EIO;
	527	goto ioremap_err;
	528	}
	529	/* allocate command queus based on cpus, max queues are 64 */
	530	ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
	531	ndev->qlen = qlen;
	532
	533	err = nitrox_pf_sw_init(ndev);
	534	if (err)
	535	goto ioremap_err;
	536
	537	err = nitrox_pf_hw_init(ndev);
	538	if (err)
	539	goto pf_hw_fail;
	540
086eac9e SJ	541	err = nitrox_debugfs_init(ndev);
	542	if (err)
	543	goto pf_hw_fail;
	544
41a9aca6	545	atomic_set(&ndev->state, __NDEV_READY);
14fa93cd SJ	546	/* barrier to sync with other cpus */
14fa93cd SJ	547	smp_mb__after_atomic();
f2663872 SJ	548
	549	err = nitrox_crypto_register();
	550	if (err)
	551	goto crypto_fail;
	552
14fa93cd SJ	553	return 0;
14fa93cd SJ	554
f2663872 SJ	555	crypto_fail:
f2663872 SJ	556	nitrox_debugfs_exit(ndev);
41a9aca6	557	atomic_set(&ndev->state, __NDEV_NOT_READY);
f2663872 SJ	558	/* barrier to sync with other cpus */
f2663872 SJ	559	smp_mb__after_atomic();
14fa93cd SJ	560	pf_hw_fail:
	561	nitrox_pf_sw_cleanup(ndev);
	562	ioremap_err:
	563	nitrox_remove_from_devlist(ndev);
	564	kfree(ndev);
	565	pci_set_drvdata(pdev, NULL);
	566	ndev_fail:
	567	pci_release_mem_regions(pdev);
	568	pci_disable_device(pdev);
	569	return err;
	570	}
	571
	572	/**
	573	* nitrox_remove - Unbind the driver from the device.
	574	* @pdev: PCI device information struct
	575	*/
	576	static void nitrox_remove(struct pci_dev *pdev)
	577	{
	578	struct nitrox_device *ndev = pci_get_drvdata(pdev);
	579
	580	if (!ndev)
	581	return;
	582
	583	if (!refcount_dec_and_test(&ndev->refcnt)) {
	584	dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
	585	refcount_read(&ndev->refcnt));
	586	return;
	587	}
	588
	589	dev_info(DEV(ndev), "Removing Device %x:%x\n",
	590	ndev->hw.vendor_id, ndev->hw.device_id);
	591
41a9aca6	592	atomic_set(&ndev->state, __NDEV_NOT_READY);
14fa93cd SJ	593	/* barrier to sync with other cpus */
	594	smp_mb__after_atomic();
	595
	596	nitrox_remove_from_devlist(ndev);
41a9aca6 SJ	597
	598	#ifdef CONFIG_PCI_IOV
	599	/* disable SR-IOV */
	600	nitrox_sriov_configure(pdev, 0);
	601	#endif
f2663872	602	nitrox_crypto_unregister();
086eac9e	603	nitrox_debugfs_exit(ndev);
14fa93cd SJ	604	nitrox_pf_sw_cleanup(ndev);
	605
	606	iounmap(ndev->bar_addr);
	607	kfree(ndev);
	608
	609	pci_set_drvdata(pdev, NULL);
	610	pci_release_mem_regions(pdev);
	611	pci_disable_device(pdev);
	612	}
	613
	614	static void nitrox_shutdown(struct pci_dev *pdev)
	615	{
	616	pci_set_drvdata(pdev, NULL);
	617	pci_release_mem_regions(pdev);
	618	pci_disable_device(pdev);
	619	}
	620
	621	static struct pci_driver nitrox_driver = {
	622	.name = nitrox_driver_name,
	623	.id_table = nitrox_pci_tbl,
	624	.probe = nitrox_probe,
	625	.remove = nitrox_remove,
	626	.shutdown = nitrox_shutdown,
41a9aca6 SJ	627	#ifdef CONFIG_PCI_IOV
	628	.sriov_configure = nitrox_sriov_configure,
	629	#endif
14fa93cd SJ	630	};
	631
	632	module_pci_driver(nitrox_driver);
	633
	634	MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
	635	MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
	636	MODULE_LICENSE("GPL");
	637	MODULE_VERSION(DRIVER_VERSION);
	638	MODULE_FIRMWARE(SE_FW);