[mirror_ubuntu-jammy-kernel.git] / drivers / crypto / cavium / nitrox / nitrox_main.c

// SPDX-License-Identifier: GPL-2.0-only
#include <linux/aer.h>
#include <linux/delay.h>
#include <linux/firmware.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/pci.h>
#include <linux/pci_ids.h>

#include "nitrox_dev.h"
#include "nitrox_common.h"
#include "nitrox_csr.h"
#include "nitrox_hal.h"
#include "nitrox_isr.h"
#include "nitrox_debugfs.h"

#define CNN55XX_DEV_ID	0x12
#define UCODE_HLEN 48
#define SE_GROUP 0

#define DRIVER_VERSION "1.1"
#define FW_DIR "cavium/"
/* SE microcode */
#define SE_FW	FW_DIR "cnn55xx_se.fw"

static const char nitrox_driver_name[] = "CNN55XX";

static LIST_HEAD(ndevlist);
static DEFINE_MUTEX(devlist_lock);
static unsigned int num_devices;

/**
 * nitrox_pci_tbl - PCI Device ID Table
 */
static const struct pci_device_id nitrox_pci_tbl[] = {
	{PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
	/* required last entry */
	{0, }
};
MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);

static unsigned int qlen = DEFAULT_CMD_QLEN;
module_param(qlen, uint, 0644);
MODULE_PARM_DESC(qlen, "Command queue length - default 2048");

#ifdef CONFIG_PCI_IOV
int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs);
#else
int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs)
{
	return 0;
}
#endif

/**
 * struct ucode - Firmware Header
 * @id: microcode ID
 * @version: firmware version
 * @code_size: code section size
 * @raz: alignment
 * @code: code section
 */
struct ucode {
	u8 id;
	char version[VERSION_LEN - 1];
	__be32 code_size;
	u8 raz[12];
	u64 code[0];
};

/**
 * write_to_ucd_unit - Write Firmware to NITROX UCD unit
 */
static void write_to_ucd_unit(struct nitrox_device *ndev,
			      struct ucode *ucode)
{
	u32 code_size = be32_to_cpu(ucode->code_size) * 2;
	u64 offset, data;
	int i = 0;

	/*
	 * UCD structure
	 *
	 *  -------------
	 *  |    BLK 7  |
	 *  -------------
	 *  |    BLK 6  |
	 *  -------------
	 *  |    ...    |
	 *  -------------
	 *  |    BLK 0  |
	 *  -------------
	 *  Total of 8 blocks, each size 32KB
	 */

	/* set the block number */
	offset = UCD_UCODE_LOAD_BLOCK_NUM;
	nitrox_write_csr(ndev, offset, 0);

	code_size = roundup(code_size, 8);
	while (code_size) {
		data = ucode->code[i];
		/* write 8 bytes at a time */
		offset = UCD_UCODE_LOAD_IDX_DATAX(i);
		nitrox_write_csr(ndev, offset, data);
		code_size -= 8;
		i++;
	}

	/* put all SE cores in group 0 */
	offset = POM_GRP_EXECMASKX(SE_GROUP);
	nitrox_write_csr(ndev, offset, (~0ULL));

	for (i = 0; i < ndev->hw.se_cores; i++) {
		/*
		 * write block number and firware length
		 * bit:<2:0> block number
		 * bit:3 is set SE uses 32KB microcode
		 * bit:3 is clear SE uses 64KB microcode
		 */
		offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
		nitrox_write_csr(ndev, offset, 0x8);
	}
	usleep_range(300, 400);
}

static int nitrox_load_fw(struct nitrox_device *ndev, const char *fw_name)
{
	const struct firmware *fw;
	struct ucode *ucode;
	int ret;

	dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);

	ret = request_firmware(&fw, fw_name, DEV(ndev));
	if (ret < 0) {
		dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
		return ret;
	}

	ucode = (struct ucode *)fw->data;
	/* copy the firmware version */
	memcpy(ndev->hw.fw_name, ucode->version, (VERSION_LEN - 2));
	ndev->hw.fw_name[VERSION_LEN - 1] = '\0';

	write_to_ucd_unit(ndev, ucode);
	release_firmware(fw);

	return 0;
}

/**
 * nitrox_add_to_devlist - add NITROX device to global device list
 * @ndev: NITROX device
 */
static int nitrox_add_to_devlist(struct nitrox_device *ndev)
{
	struct nitrox_device *dev;
	int ret = 0;

	INIT_LIST_HEAD(&ndev->list);
	refcount_set(&ndev->refcnt, 1);

	mutex_lock(&devlist_lock);
	list_for_each_entry(dev, &ndevlist, list) {
		if (dev == ndev) {
			ret = -EEXIST;
			goto unlock;
		}
	}
	ndev->idx = num_devices++;
	list_add_tail(&ndev->list, &ndevlist);
unlock:
	mutex_unlock(&devlist_lock);
	return ret;
}

/**
 * nitrox_remove_from_devlist - remove NITROX device from
 *   global device list
 * @ndev: NITROX device
 */
static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
{
	mutex_lock(&devlist_lock);
	list_del(&ndev->list);
	num_devices--;
	mutex_unlock(&devlist_lock);
}

struct nitrox_device *nitrox_get_first_device(void)
{
	struct nitrox_device *ndev = NULL;

	mutex_lock(&devlist_lock);
	list_for_each_entry(ndev, &ndevlist, list) {
		if (nitrox_ready(ndev))
			break;
	}
	mutex_unlock(&devlist_lock);
	if (!ndev)
		return NULL;

	refcount_inc(&ndev->refcnt);
	/* barrier to sync with other cpus */
	smp_mb__after_atomic();
	return ndev;
}

void nitrox_put_device(struct nitrox_device *ndev)
{
	if (!ndev)
		return;

	refcount_dec(&ndev->refcnt);
	/* barrier to sync with other cpus */
	smp_mb__after_atomic();
}

static int nitrox_device_flr(struct pci_dev *pdev)
{
	int pos = 0;

	pos = pci_save_state(pdev);
	if (pos) {
		dev_err(&pdev->dev, "Failed to save pci state\n");
		return -ENOMEM;
	}

	/* check flr support */
	if (pcie_has_flr(pdev))
		pcie_flr(pdev);

	pci_restore_state(pdev);

	return 0;
}

static int nitrox_pf_sw_init(struct nitrox_device *ndev)
{
	int err;

	err = nitrox_common_sw_init(ndev);
	if (err)
		return err;

	err = nitrox_register_interrupts(ndev);
	if (err)
		nitrox_common_sw_cleanup(ndev);

	return err;
}

static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
{
	nitrox_unregister_interrupts(ndev);
	nitrox_common_sw_cleanup(ndev);
}

/**
 * nitrox_bist_check - Check NITORX BIST registers status
 * @ndev: NITROX device
 */
static int nitrox_bist_check(struct nitrox_device *ndev)
{
	u64 value = 0;
	int i;

	for (i = 0; i < NR_CLUSTERS; i++) {
		value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
		value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
	}
	value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
	value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
	value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
	value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
	value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
	value += nitrox_read_csr(ndev, POM_BIST_REG);
	value += nitrox_read_csr(ndev, BMI_BIST_REG);
	value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
	value += nitrox_read_csr(ndev, BMO_BIST_REG);
	value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
	value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
	if (value)
		return -EIO;
	return 0;
}

static int nitrox_pf_hw_init(struct nitrox_device *ndev)
{
	int err;

	err = nitrox_bist_check(ndev);
	if (err) {
		dev_err(&ndev->pdev->dev, "BIST check failed\n");
		return err;
	}
	/* get cores information */
	nitrox_get_hwinfo(ndev);

	nitrox_config_nps_unit(ndev);
	nitrox_config_pom_unit(ndev);
	nitrox_config_efl_unit(ndev);
	/* configure IO units */
	nitrox_config_bmi_unit(ndev);
	nitrox_config_bmo_unit(ndev);
	/* configure Local Buffer Cache */
	nitrox_config_lbc_unit(ndev);
	nitrox_config_rand_unit(ndev);

	/* load firmware on SE cores */
	err = nitrox_load_fw(ndev, SE_FW);
	if (err)
		return err;

	nitrox_config_emu_unit(ndev);

	return 0;
}

/**
 * nitrox_probe - NITROX Initialization function.
 * @pdev: PCI device information struct
 * @id: entry in nitrox_pci_tbl
 *
 * Return: 0, if the driver is bound to the device, or
 *         a negative error if there is failure.
 */
static int nitrox_probe(struct pci_dev *pdev,
			const struct pci_device_id *id)
{
	struct nitrox_device *ndev;
	int err;

	dev_info_once(&pdev->dev, "%s driver version %s\n",
		      nitrox_driver_name, DRIVER_VERSION);

	err = pci_enable_device_mem(pdev);
	if (err)
		return err;

	/* do FLR */
	err = nitrox_device_flr(pdev);
	if (err) {
		dev_err(&pdev->dev, "FLR failed\n");
		pci_disable_device(pdev);
		return err;
	}

	if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
		dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
	} else {
		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
		if (err) {
			dev_err(&pdev->dev, "DMA configuration failed\n");
			pci_disable_device(pdev);
			return err;
		}
	}

	err = pci_request_mem_regions(pdev, nitrox_driver_name);
	if (err) {
		pci_disable_device(pdev);
		return err;
	}
	pci_set_master(pdev);

	ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
	if (!ndev) {
		err = -ENOMEM;
		goto ndev_fail;
	}

	pci_set_drvdata(pdev, ndev);
	ndev->pdev = pdev;

	/* add to device list */
	nitrox_add_to_devlist(ndev);

	ndev->hw.vendor_id = pdev->vendor;
	ndev->hw.device_id = pdev->device;
	ndev->hw.revision_id = pdev->revision;
	/* command timeout in jiffies */
	ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
	ndev->node = dev_to_node(&pdev->dev);
	if (ndev->node == NUMA_NO_NODE)
		ndev->node = 0;

	ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
				 pci_resource_len(pdev, 0));
	if (!ndev->bar_addr) {
		err = -EIO;
		goto ioremap_err;
	}
	/* allocate command queus based on cpus, max queues are 64 */
	ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
	ndev->qlen = qlen;

	err = nitrox_pf_sw_init(ndev);
	if (err)
		goto ioremap_err;

	err = nitrox_pf_hw_init(ndev);
	if (err)
		goto pf_hw_fail;

	nitrox_debugfs_init(ndev);

	/* clear the statistics */
	atomic64_set(&ndev->stats.posted, 0);
	atomic64_set(&ndev->stats.completed, 0);
	atomic64_set(&ndev->stats.dropped, 0);

	atomic_set(&ndev->state, __NDEV_READY);
	/* barrier to sync with other cpus */
	smp_mb__after_atomic();

	err = nitrox_crypto_register();
	if (err)
		goto crypto_fail;

	return 0;

crypto_fail:
	nitrox_debugfs_exit(ndev);
	atomic_set(&ndev->state, __NDEV_NOT_READY);
	/* barrier to sync with other cpus */
	smp_mb__after_atomic();
pf_hw_fail:
	nitrox_pf_sw_cleanup(ndev);
ioremap_err:
	nitrox_remove_from_devlist(ndev);
	kfree(ndev);
	pci_set_drvdata(pdev, NULL);
ndev_fail:
	pci_release_mem_regions(pdev);
	pci_disable_device(pdev);
	return err;
}

/**
 * nitrox_remove - Unbind the driver from the device.
 * @pdev: PCI device information struct
 */
static void nitrox_remove(struct pci_dev *pdev)
{
	struct nitrox_device *ndev = pci_get_drvdata(pdev);

	if (!ndev)
		return;

	if (!refcount_dec_and_test(&ndev->refcnt)) {
		dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
			refcount_read(&ndev->refcnt));
		return;
	}

	dev_info(DEV(ndev), "Removing Device %x:%x\n",
		 ndev->hw.vendor_id, ndev->hw.device_id);

	atomic_set(&ndev->state, __NDEV_NOT_READY);
	/* barrier to sync with other cpus */
	smp_mb__after_atomic();

	nitrox_remove_from_devlist(ndev);

#ifdef CONFIG_PCI_IOV
	/* disable SR-IOV */
	nitrox_sriov_configure(pdev, 0);
#endif
	nitrox_crypto_unregister();
	nitrox_debugfs_exit(ndev);
	nitrox_pf_sw_cleanup(ndev);

	iounmap(ndev->bar_addr);
	kfree(ndev);

	pci_set_drvdata(pdev, NULL);
	pci_release_mem_regions(pdev);
	pci_disable_device(pdev);
}

static void nitrox_shutdown(struct pci_dev *pdev)
{
	pci_set_drvdata(pdev, NULL);
	pci_release_mem_regions(pdev);
	pci_disable_device(pdev);
}

static struct pci_driver nitrox_driver = {
	.name = nitrox_driver_name,
	.id_table = nitrox_pci_tbl,
	.probe = nitrox_probe,
	.remove	= nitrox_remove,
	.shutdown = nitrox_shutdown,
#ifdef CONFIG_PCI_IOV
	.sriov_configure = nitrox_sriov_configure,
#endif
};

module_pci_driver(nitrox_driver);

MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
MODULE_LICENSE("GPL");
MODULE_VERSION(DRIVER_VERSION);
MODULE_FIRMWARE(SE_FW);
Commit	Line	Data
09c434b8	1	// SPDX-License-Identifier: GPL-2.0-only
14fa93cd SJ	2	#include <linux/aer.h>
	3	#include <linux/delay.h>
	4	#include <linux/firmware.h>
	5	#include <linux/list.h>
	6	#include <linux/module.h>
	7	#include <linux/mutex.h>
	8	#include <linux/pci.h>
	9	#include <linux/pci_ids.h>
	10
	11	#include "nitrox_dev.h"
	12	#include "nitrox_common.h"
	13	#include "nitrox_csr.h"
41a9aca6	14	#include "nitrox_hal.h"
5155e118	15	#include "nitrox_isr.h"
cf718eaa	16	#include "nitrox_debugfs.h"
14fa93cd SJ	17
14fa93cd SJ	18	#define CNN55XX_DEV_ID 0x12
14fa93cd SJ	19	#define UCODE_HLEN 48
	20	#define SE_GROUP 0
	21
41a9aca6	22	#define DRIVER_VERSION "1.1"
440bf347	23	#define FW_DIR "cavium/"
14fa93cd	24	/* SE microcode */
440bf347	25	#define SE_FW FW_DIR "cnn55xx_se.fw"
14fa93cd SJ	26
	27	static const char nitrox_driver_name[] = "CNN55XX";
	28
	29	static LIST_HEAD(ndevlist);
	30	static DEFINE_MUTEX(devlist_lock);
	31	static unsigned int num_devices;
	32
	33	/**
	34	* nitrox_pci_tbl - PCI Device ID Table
	35	*/
	36	static const struct pci_device_id nitrox_pci_tbl[] = {
	37	{PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
	38	/* required last entry */
	39	{0, }
	40	};
	41	MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);
	42
	43	static unsigned int qlen = DEFAULT_CMD_QLEN;
	44	module_param(qlen, uint, 0644);
	45	MODULE_PARM_DESC(qlen, "Command queue length - default 2048");
	46
41a9aca6 SJ	47	#ifdef CONFIG_PCI_IOV
	48	int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs);
	49	#else
	50	int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs)
	51	{
	52	return 0;
	53	}
	54	#endif
	55
14fa93cd SJ	56	/**
	57	* struct ucode - Firmware Header
	58	* @id: microcode ID
	59	* @version: firmware version
	60	* @code_size: code section size
	61	* @raz: alignment
	62	* @code: code section
	63	*/
	64	struct ucode {
	65	u8 id;
	66	char version[VERSION_LEN - 1];
	67	__be32 code_size;
	68	u8 raz[12];
	69	u64 code[0];
	70	};
	71
	72	/**
	73	* write_to_ucd_unit - Write Firmware to NITROX UCD unit
	74	*/
	75	static void write_to_ucd_unit(struct nitrox_device *ndev,
	76	struct ucode *ucode)
	77	{
	78	u32 code_size = be32_to_cpu(ucode->code_size) * 2;
	79	u64 offset, data;
	80	int i = 0;
	81
	82	/*
	83	* UCD structure
	84	*
	85	* -------------
	86	* \| BLK 7 \|
	87	* -------------
	88	* \| BLK 6 \|
	89	* -------------
	90	* \| ... \|
	91	* -------------
	92	* \| BLK 0 \|
	93	* -------------
	94	* Total of 8 blocks, each size 32KB
	95	*/
	96
	97	/* set the block number */
	98	offset = UCD_UCODE_LOAD_BLOCK_NUM;
	99	nitrox_write_csr(ndev, offset, 0);
	100
	101	code_size = roundup(code_size, 8);
	102	while (code_size) {
	103	data = ucode->code[i];
	104	/* write 8 bytes at a time */
	105	offset = UCD_UCODE_LOAD_IDX_DATAX(i);
	106	nitrox_write_csr(ndev, offset, data);
	107	code_size -= 8;
	108	i++;
	109	}
	110
	111	/* put all SE cores in group 0 */
	112	offset = POM_GRP_EXECMASKX(SE_GROUP);
	113	nitrox_write_csr(ndev, offset, (~0ULL));
	114
	115	for (i = 0; i < ndev->hw.se_cores; i++) {
	116	/*
	117	* write block number and firware length
	118	* bit:<2:0> block number
	119	* bit:3 is set SE uses 32KB microcode
120	* bit:3 is clear SE uses 64KB microcode
121	*/
122	offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
123	nitrox_write_csr(ndev, offset, 0x8);
124	}
125	usleep_range(300, 400);
126	}
127
128	static int nitrox_load_fw(struct nitrox_device ndev, const char fw_name)
129	{
130	const struct firmware *fw;
131	struct ucode *ucode;
132	int ret;
133
134	dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
135
136	ret = request_firmware(&fw, fw_name, DEV(ndev));
137	if (ret < 0) {
138	dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
139	return ret;
140	}
141
142	ucode = (struct ucode *)fw->data;
143	/* copy the firmware version */
144	memcpy(ndev->hw.fw_name, ucode->version, (VERSION_LEN - 2));
145	ndev->hw.fw_name[VERSION_LEN - 1] = '\0';
146
147	write_to_ucd_unit(ndev, ucode);
148	release_firmware(fw);
149
14fa93cd SJ	150	return 0;
	151	}
	152
	153	/**
	154	* nitrox_add_to_devlist - add NITROX device to global device list
	155	* @ndev: NITROX device
	156	*/
	157	static int nitrox_add_to_devlist(struct nitrox_device *ndev)
	158	{
	159	struct nitrox_device *dev;
	160	int ret = 0;
	161
	162	INIT_LIST_HEAD(&ndev->list);
	163	refcount_set(&ndev->refcnt, 1);
	164
	165	mutex_lock(&devlist_lock);
	166	list_for_each_entry(dev, &ndevlist, list) {
	167	if (dev == ndev) {
	168	ret = -EEXIST;
	169	goto unlock;
	170	}
	171	}
	172	ndev->idx = num_devices++;
	173	list_add_tail(&ndev->list, &ndevlist);
	174	unlock:
	175	mutex_unlock(&devlist_lock);
	176	return ret;
	177	}
	178
	179	/**
	180	* nitrox_remove_from_devlist - remove NITROX device from
	181	* global device list
	182	* @ndev: NITROX device
	183	*/
	184	static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
	185	{
	186	mutex_lock(&devlist_lock);
	187	list_del(&ndev->list);
	188	num_devices--;
	189	mutex_unlock(&devlist_lock);
	190	}
	191
f2663872 SJ	192	struct nitrox_device *nitrox_get_first_device(void)
	193	{
	194	struct nitrox_device *ndev = NULL;
	195
	196	mutex_lock(&devlist_lock);
	197	list_for_each_entry(ndev, &ndevlist, list) {
	198	if (nitrox_ready(ndev))
	199	break;
	200	}
	201	mutex_unlock(&devlist_lock);
	202	if (!ndev)
	203	return NULL;
	204
	205	refcount_inc(&ndev->refcnt);
	206	/* barrier to sync with other cpus */
	207	smp_mb__after_atomic();
	208	return ndev;
	209	}
	210
	211	void nitrox_put_device(struct nitrox_device *ndev)
	212	{
	213	if (!ndev)
	214	return;
	215
	216	refcount_dec(&ndev->refcnt);
	217	/* barrier to sync with other cpus */
	218	smp_mb__after_atomic();
	219	}
	220
0b501e7a	221	static int nitrox_device_flr(struct pci_dev *pdev)
14fa93cd SJ	222	{
	223	int pos = 0;
	224
	225	pos = pci_save_state(pdev);
	226	if (pos) {
	227	dev_err(&pdev->dev, "Failed to save pci state\n");
	228	return -ENOMEM;
	229	}
	230
0b501e7a SJ	231	/* check flr support */
	232	if (pcie_has_flr(pdev))
	233	pcie_flr(pdev);
14fa93cd	234
14fa93cd SJ	235	pci_restore_state(pdev);
	236
	237	return 0;
	238	}
	239
	240	static int nitrox_pf_sw_init(struct nitrox_device *ndev)
	241	{
	242	int err;
	243
	244	err = nitrox_common_sw_init(ndev);
	245	if (err)
	246	return err;
	247
5155e118	248	err = nitrox_register_interrupts(ndev);
14fa93cd SJ	249	if (err)
	250	nitrox_common_sw_cleanup(ndev);
	251
	252	return err;
	253	}
	254
	255	static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
	256	{
5155e118	257	nitrox_unregister_interrupts(ndev);
14fa93cd SJ	258	nitrox_common_sw_cleanup(ndev);
	259	}
	260
	261	/**
	262	* nitrox_bist_check - Check NITORX BIST registers status
	263	* @ndev: NITROX device
	264	*/
	265	static int nitrox_bist_check(struct nitrox_device *ndev)
	266	{
	267	u64 value = 0;
	268	int i;
	269
	270	for (i = 0; i < NR_CLUSTERS; i++) {
	271	value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
	272	value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
	273	}
	274	value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
	275	value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
	276	value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
	277	value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
	278	value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
	279	value += nitrox_read_csr(ndev, POM_BIST_REG);
	280	value += nitrox_read_csr(ndev, BMI_BIST_REG);
	281	value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
	282	value += nitrox_read_csr(ndev, BMO_BIST_REG);
	283	value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
	284	value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
	285	if (value)
	286	return -EIO;
	287	return 0;
	288	}
	289
14fa93cd SJ	290	static int nitrox_pf_hw_init(struct nitrox_device *ndev)
	291	{
	292	int err;
	293
	294	err = nitrox_bist_check(ndev);
	295	if (err) {
	296	dev_err(&ndev->pdev->dev, "BIST check failed\n");
	297	return err;
	298	}
	299	/* get cores information */
	300	nitrox_get_hwinfo(ndev);
	301
	302	nitrox_config_nps_unit(ndev);
	303	nitrox_config_pom_unit(ndev);
	304	nitrox_config_efl_unit(ndev);
	305	/* configure IO units */
	306	nitrox_config_bmi_unit(ndev);
	307	nitrox_config_bmo_unit(ndev);
	308	/* configure Local Buffer Cache */
	309	nitrox_config_lbc_unit(ndev);
	310	nitrox_config_rand_unit(ndev);
	311
	312	/* load firmware on SE cores */
	313	err = nitrox_load_fw(ndev, SE_FW);
	314	if (err)
	315	return err;
	316
	317	nitrox_config_emu_unit(ndev);
	318
	319	return 0;
	320	}
	321
	322	/**
	323	* nitrox_probe - NITROX Initialization function.
	324	* @pdev: PCI device information struct
	325	* @id: entry in nitrox_pci_tbl
	326	*
	327	* Return: 0, if the driver is bound to the device, or
	328	* a negative error if there is failure.
	329	*/
	330	static int nitrox_probe(struct pci_dev *pdev,
	331	const struct pci_device_id *id)
	332	{
	333	struct nitrox_device *ndev;
	334	int err;
	335
	336	dev_info_once(&pdev->dev, "%s driver version %s\n",
	337	nitrox_driver_name, DRIVER_VERSION);
	338
	339	err = pci_enable_device_mem(pdev);
	340	if (err)
	341	return err;
	342
	343	/* do FLR */
0b501e7a	344	err = nitrox_device_flr(pdev);
14fa93cd SJ	345	if (err) {
	346	dev_err(&pdev->dev, "FLR failed\n");
	347	pci_disable_device(pdev);
	348	return err;
	349	}
	350
	351	if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
	352	dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
	353	} else {
	354	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
	355	if (err) {
	356	dev_err(&pdev->dev, "DMA configuration failed\n");
	357	pci_disable_device(pdev);
	358	return err;
	359	}
	360	}
	361
	362	err = pci_request_mem_regions(pdev, nitrox_driver_name);
	363	if (err) {
	364	pci_disable_device(pdev);
	365	return err;
	366	}
	367	pci_set_master(pdev);
	368
	369	ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
32e67b9a CJ	370	if (!ndev) {
32e67b9a CJ	371	err = -ENOMEM;
14fa93cd	372	goto ndev_fail;
32e67b9a	373	}
14fa93cd SJ	374
	375	pci_set_drvdata(pdev, ndev);
	376	ndev->pdev = pdev;
	377
	378	/* add to device list */
	379	nitrox_add_to_devlist(ndev);
	380
	381	ndev->hw.vendor_id = pdev->vendor;
	382	ndev->hw.device_id = pdev->device;
	383	ndev->hw.revision_id = pdev->revision;
	384	/* command timeout in jiffies */
	385	ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
	386	ndev->node = dev_to_node(&pdev->dev);
	387	if (ndev->node == NUMA_NO_NODE)
	388	ndev->node = 0;
	389
	390	ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
	391	pci_resource_len(pdev, 0));
	392	if (!ndev->bar_addr) {
	393	err = -EIO;
	394	goto ioremap_err;
	395	}
	396	/* allocate command queus based on cpus, max queues are 64 */
	397	ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
	398	ndev->qlen = qlen;
	399
	400	err = nitrox_pf_sw_init(ndev);
	401	if (err)
	402	goto ioremap_err;
	403
	404	err = nitrox_pf_hw_init(ndev);
	405	if (err)
	406	goto pf_hw_fail;
	407
97a93b2b	408	nitrox_debugfs_init(ndev);
086eac9e	409
fec165c9 SJ	410	/* clear the statistics */
	411	atomic64_set(&ndev->stats.posted, 0);
	412	atomic64_set(&ndev->stats.completed, 0);
	413	atomic64_set(&ndev->stats.dropped, 0);
	414
41a9aca6	415	atomic_set(&ndev->state, __NDEV_READY);
14fa93cd SJ	416	/* barrier to sync with other cpus */
14fa93cd SJ	417	smp_mb__after_atomic();
f2663872 SJ	418
	419	err = nitrox_crypto_register();
	420	if (err)
	421	goto crypto_fail;
	422
14fa93cd SJ	423	return 0;
14fa93cd SJ	424
f2663872 SJ	425	crypto_fail:
f2663872 SJ	426	nitrox_debugfs_exit(ndev);
41a9aca6	427	atomic_set(&ndev->state, __NDEV_NOT_READY);
f2663872 SJ	428	/* barrier to sync with other cpus */
f2663872 SJ	429	smp_mb__after_atomic();
14fa93cd SJ	430	pf_hw_fail:
	431	nitrox_pf_sw_cleanup(ndev);
	432	ioremap_err:
	433	nitrox_remove_from_devlist(ndev);
	434	kfree(ndev);
	435	pci_set_drvdata(pdev, NULL);
	436	ndev_fail:
	437	pci_release_mem_regions(pdev);
	438	pci_disable_device(pdev);
	439	return err;
	440	}
	441
	442	/**
	443	* nitrox_remove - Unbind the driver from the device.
	444	* @pdev: PCI device information struct
	445	*/
	446	static void nitrox_remove(struct pci_dev *pdev)
	447	{
	448	struct nitrox_device *ndev = pci_get_drvdata(pdev);
	449
	450	if (!ndev)
	451	return;
	452
	453	if (!refcount_dec_and_test(&ndev->refcnt)) {
	454	dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
	455	refcount_read(&ndev->refcnt));
	456	return;
	457	}
	458
	459	dev_info(DEV(ndev), "Removing Device %x:%x\n",
	460	ndev->hw.vendor_id, ndev->hw.device_id);
	461
41a9aca6	462	atomic_set(&ndev->state, __NDEV_NOT_READY);
14fa93cd SJ	463	/* barrier to sync with other cpus */
	464	smp_mb__after_atomic();
	465
	466	nitrox_remove_from_devlist(ndev);
41a9aca6 SJ	467
	468	#ifdef CONFIG_PCI_IOV
	469	/* disable SR-IOV */
	470	nitrox_sriov_configure(pdev, 0);
	471	#endif
f2663872	472	nitrox_crypto_unregister();
086eac9e	473	nitrox_debugfs_exit(ndev);
14fa93cd SJ	474	nitrox_pf_sw_cleanup(ndev);
	475
	476	iounmap(ndev->bar_addr);
	477	kfree(ndev);
	478
	479	pci_set_drvdata(pdev, NULL);
	480	pci_release_mem_regions(pdev);
	481	pci_disable_device(pdev);
	482	}
	483
	484	static void nitrox_shutdown(struct pci_dev *pdev)
	485	{
	486	pci_set_drvdata(pdev, NULL);
	487	pci_release_mem_regions(pdev);
	488	pci_disable_device(pdev);
	489	}
	490
	491	static struct pci_driver nitrox_driver = {
	492	.name = nitrox_driver_name,
	493	.id_table = nitrox_pci_tbl,
	494	.probe = nitrox_probe,
	495	.remove = nitrox_remove,
	496	.shutdown = nitrox_shutdown,
41a9aca6 SJ	497	#ifdef CONFIG_PCI_IOV
	498	.sriov_configure = nitrox_sriov_configure,
	499	#endif
14fa93cd SJ	500	};
	501
	502	module_pci_driver(nitrox_driver);
	503
	504	MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
	505	MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
	506	MODULE_LICENSE("GPL");
	507	MODULE_VERSION(DRIVER_VERSION);
	508	MODULE_FIRMWARE(SE_FW);