[mirror_ubuntu-jammy-kernel.git] / drivers / crypto / cavium / nitrox / nitrox_main.c

#include <linux/aer.h>
#include <linux/delay.h>
#include <linux/debugfs.h>
#include <linux/firmware.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/pci.h>
#include <linux/pci_ids.h>

#include "nitrox_dev.h"
#include "nitrox_common.h"
#include "nitrox_csr.h"
#include "nitrox_hal.h"

#define CNN55XX_DEV_ID	0x12
#define MAX_PF_QUEUES	64
#define UCODE_HLEN 48
#define SE_GROUP 0

#define DRIVER_VERSION "1.1"
#define FW_DIR "cavium/"
/* SE microcode */
#define SE_FW	FW_DIR "cnn55xx_se.fw"

static const char nitrox_driver_name[] = "CNN55XX";

static LIST_HEAD(ndevlist);
static DEFINE_MUTEX(devlist_lock);
static unsigned int num_devices;

/**
 * nitrox_pci_tbl - PCI Device ID Table
 */
static const struct pci_device_id nitrox_pci_tbl[] = {
	{PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
	/* required last entry */
	{0, }
};
MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);

static unsigned int qlen = DEFAULT_CMD_QLEN;
module_param(qlen, uint, 0644);
MODULE_PARM_DESC(qlen, "Command queue length - default 2048");

#ifdef CONFIG_PCI_IOV
int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs);
#else
int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs)
{
	return 0;
}
#endif

/**
 * struct ucode - Firmware Header
 * @id: microcode ID
 * @version: firmware version
 * @code_size: code section size
 * @raz: alignment
 * @code: code section
 */
struct ucode {
	u8 id;
	char version[VERSION_LEN - 1];
	__be32 code_size;
	u8 raz[12];
	u64 code[0];
};

/**
 * write_to_ucd_unit - Write Firmware to NITROX UCD unit
 */
static void write_to_ucd_unit(struct nitrox_device *ndev,
			      struct ucode *ucode)
{
	u32 code_size = be32_to_cpu(ucode->code_size) * 2;
	u64 offset, data;
	int i = 0;

	/*
	 * UCD structure
	 *
	 *  -------------
	 *  |    BLK 7  |
	 *  -------------
	 *  |    BLK 6  |
	 *  -------------
	 *  |    ...    |
	 *  -------------
	 *  |    BLK 0  |
	 *  -------------
	 *  Total of 8 blocks, each size 32KB
	 */

	/* set the block number */
	offset = UCD_UCODE_LOAD_BLOCK_NUM;
	nitrox_write_csr(ndev, offset, 0);

	code_size = roundup(code_size, 8);
	while (code_size) {
		data = ucode->code[i];
		/* write 8 bytes at a time */
		offset = UCD_UCODE_LOAD_IDX_DATAX(i);
		nitrox_write_csr(ndev, offset, data);
		code_size -= 8;
		i++;
	}

	/* put all SE cores in group 0 */
	offset = POM_GRP_EXECMASKX(SE_GROUP);
	nitrox_write_csr(ndev, offset, (~0ULL));

	for (i = 0; i < ndev->hw.se_cores; i++) {
		/*
		 * write block number and firware length
		 * bit:<2:0> block number
		 * bit:3 is set SE uses 32KB microcode
		 * bit:3 is clear SE uses 64KB microcode
		 */
		offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
		nitrox_write_csr(ndev, offset, 0x8);
	}
	usleep_range(300, 400);
}

static int nitrox_load_fw(struct nitrox_device *ndev, const char *fw_name)
{
	const struct firmware *fw;
	struct ucode *ucode;
	int ret;

	dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);

	ret = request_firmware(&fw, fw_name, DEV(ndev));
	if (ret < 0) {
		dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
		return ret;
	}

	ucode = (struct ucode *)fw->data;
	/* copy the firmware version */
	memcpy(ndev->hw.fw_name, ucode->version, (VERSION_LEN - 2));
	ndev->hw.fw_name[VERSION_LEN - 1] = '\0';

	write_to_ucd_unit(ndev, ucode);
	release_firmware(fw);

	return 0;
}

/**
 * nitrox_add_to_devlist - add NITROX device to global device list
 * @ndev: NITROX device
 */
static int nitrox_add_to_devlist(struct nitrox_device *ndev)
{
	struct nitrox_device *dev;
	int ret = 0;

	INIT_LIST_HEAD(&ndev->list);
	refcount_set(&ndev->refcnt, 1);

	mutex_lock(&devlist_lock);
	list_for_each_entry(dev, &ndevlist, list) {
		if (dev == ndev) {
			ret = -EEXIST;
			goto unlock;
		}
	}
	ndev->idx = num_devices++;
	list_add_tail(&ndev->list, &ndevlist);
unlock:
	mutex_unlock(&devlist_lock);
	return ret;
}

/**
 * nitrox_remove_from_devlist - remove NITROX device from
 *   global device list
 * @ndev: NITROX device
 */
static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
{
	mutex_lock(&devlist_lock);
	list_del(&ndev->list);
	num_devices--;
	mutex_unlock(&devlist_lock);
}

struct nitrox_device *nitrox_get_first_device(void)
{
	struct nitrox_device *ndev = NULL;

	mutex_lock(&devlist_lock);
	list_for_each_entry(ndev, &ndevlist, list) {
		if (nitrox_ready(ndev))
			break;
	}
	mutex_unlock(&devlist_lock);
	if (!ndev)
		return NULL;

	refcount_inc(&ndev->refcnt);
	/* barrier to sync with other cpus */
	smp_mb__after_atomic();
	return ndev;
}

void nitrox_put_device(struct nitrox_device *ndev)
{
	if (!ndev)
		return;

	refcount_dec(&ndev->refcnt);
	/* barrier to sync with other cpus */
	smp_mb__after_atomic();
}

static int nitrox_reset_device(struct pci_dev *pdev)
{
	int pos = 0;

	pos = pci_save_state(pdev);
	if (pos) {
		dev_err(&pdev->dev, "Failed to save pci state\n");
		return -ENOMEM;
	}

	pos = pci_pcie_cap(pdev);
	if (!pos)
		return -ENOTTY;

	if (!pci_wait_for_pending_transaction(pdev))
		dev_err(&pdev->dev, "waiting for pending transaction\n");

	pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
	msleep(100);
	pci_restore_state(pdev);

	return 0;
}

static int nitrox_pf_sw_init(struct nitrox_device *ndev)
{
	int err;

	err = nitrox_common_sw_init(ndev);
	if (err)
		return err;

	err = nitrox_pf_init_isr(ndev);
	if (err)
		nitrox_common_sw_cleanup(ndev);

	return err;
}

static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
{
	nitrox_pf_cleanup_isr(ndev);
	nitrox_common_sw_cleanup(ndev);
}

/**
 * nitrox_bist_check - Check NITORX BIST registers status
 * @ndev: NITROX device
 */
static int nitrox_bist_check(struct nitrox_device *ndev)
{
	u64 value = 0;
	int i;

	for (i = 0; i < NR_CLUSTERS; i++) {
		value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
		value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
	}
	value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
	value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
	value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
	value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
	value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
	value += nitrox_read_csr(ndev, POM_BIST_REG);
	value += nitrox_read_csr(ndev, BMI_BIST_REG);
	value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
	value += nitrox_read_csr(ndev, BMO_BIST_REG);
	value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
	value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
	if (value)
		return -EIO;
	return 0;
}

static int nitrox_pf_hw_init(struct nitrox_device *ndev)
{
	int err;

	err = nitrox_bist_check(ndev);
	if (err) {
		dev_err(&ndev->pdev->dev, "BIST check failed\n");
		return err;
	}
	/* get cores information */
	nitrox_get_hwinfo(ndev);

	nitrox_config_nps_unit(ndev);
	nitrox_config_pom_unit(ndev);
	nitrox_config_efl_unit(ndev);
	/* configure IO units */
	nitrox_config_bmi_unit(ndev);
	nitrox_config_bmo_unit(ndev);
	/* configure Local Buffer Cache */
	nitrox_config_lbc_unit(ndev);
	nitrox_config_rand_unit(ndev);

	/* load firmware on SE cores */
	err = nitrox_load_fw(ndev, SE_FW);
	if (err)
		return err;

	nitrox_config_emu_unit(ndev);

	return 0;
}

/**
 * nitrox_probe - NITROX Initialization function.
 * @pdev: PCI device information struct
 * @id: entry in nitrox_pci_tbl
 *
 * Return: 0, if the driver is bound to the device, or
 *         a negative error if there is failure.
 */
static int nitrox_probe(struct pci_dev *pdev,
			const struct pci_device_id *id)
{
	struct nitrox_device *ndev;
	int err;

	dev_info_once(&pdev->dev, "%s driver version %s\n",
		      nitrox_driver_name, DRIVER_VERSION);

	err = pci_enable_device_mem(pdev);
	if (err)
		return err;

	/* do FLR */
	err = nitrox_reset_device(pdev);
	if (err) {
		dev_err(&pdev->dev, "FLR failed\n");
		pci_disable_device(pdev);
		return err;
	}

	if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
		dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
	} else {
		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
		if (err) {
			dev_err(&pdev->dev, "DMA configuration failed\n");
			pci_disable_device(pdev);
			return err;
		}
	}

	err = pci_request_mem_regions(pdev, nitrox_driver_name);
	if (err) {
		pci_disable_device(pdev);
		return err;
	}
	pci_set_master(pdev);

	ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
	if (!ndev) {
		err = -ENOMEM;
		goto ndev_fail;
	}

	pci_set_drvdata(pdev, ndev);
	ndev->pdev = pdev;

	/* add to device list */
	nitrox_add_to_devlist(ndev);

	ndev->hw.vendor_id = pdev->vendor;
	ndev->hw.device_id = pdev->device;
	ndev->hw.revision_id = pdev->revision;
	/* command timeout in jiffies */
	ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
	ndev->node = dev_to_node(&pdev->dev);
	if (ndev->node == NUMA_NO_NODE)
		ndev->node = 0;

	ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
				 pci_resource_len(pdev, 0));
	if (!ndev->bar_addr) {
		err = -EIO;
		goto ioremap_err;
	}
	/* allocate command queus based on cpus, max queues are 64 */
	ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
	ndev->qlen = qlen;

	err = nitrox_pf_sw_init(ndev);
	if (err)
		goto ioremap_err;

	err = nitrox_pf_hw_init(ndev);
	if (err)
		goto pf_hw_fail;

	err = nitrox_debugfs_init(ndev);
	if (err)
		goto pf_hw_fail;

	/* clear the statistics */
	atomic64_set(&ndev->stats.posted, 0);
	atomic64_set(&ndev->stats.completed, 0);
	atomic64_set(&ndev->stats.dropped, 0);

	atomic_set(&ndev->state, __NDEV_READY);
	/* barrier to sync with other cpus */
	smp_mb__after_atomic();

	err = nitrox_crypto_register();
	if (err)
		goto crypto_fail;

	return 0;

crypto_fail:
	nitrox_debugfs_exit(ndev);
	atomic_set(&ndev->state, __NDEV_NOT_READY);
	/* barrier to sync with other cpus */
	smp_mb__after_atomic();
pf_hw_fail:
	nitrox_pf_sw_cleanup(ndev);
ioremap_err:
	nitrox_remove_from_devlist(ndev);
	kfree(ndev);
	pci_set_drvdata(pdev, NULL);
ndev_fail:
	pci_release_mem_regions(pdev);
	pci_disable_device(pdev);
	return err;
}

/**
 * nitrox_remove - Unbind the driver from the device.
 * @pdev: PCI device information struct
 */
static void nitrox_remove(struct pci_dev *pdev)
{
	struct nitrox_device *ndev = pci_get_drvdata(pdev);

	if (!ndev)
		return;

	if (!refcount_dec_and_test(&ndev->refcnt)) {
		dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
			refcount_read(&ndev->refcnt));
		return;
	}

	dev_info(DEV(ndev), "Removing Device %x:%x\n",
		 ndev->hw.vendor_id, ndev->hw.device_id);

	atomic_set(&ndev->state, __NDEV_NOT_READY);
	/* barrier to sync with other cpus */
	smp_mb__after_atomic();

	nitrox_remove_from_devlist(ndev);

#ifdef CONFIG_PCI_IOV
	/* disable SR-IOV */
	nitrox_sriov_configure(pdev, 0);
#endif
	nitrox_crypto_unregister();
	nitrox_debugfs_exit(ndev);
	nitrox_pf_sw_cleanup(ndev);

	iounmap(ndev->bar_addr);
	kfree(ndev);

	pci_set_drvdata(pdev, NULL);
	pci_release_mem_regions(pdev);
	pci_disable_device(pdev);
}

static void nitrox_shutdown(struct pci_dev *pdev)
{
	pci_set_drvdata(pdev, NULL);
	pci_release_mem_regions(pdev);
	pci_disable_device(pdev);
}

static struct pci_driver nitrox_driver = {
	.name = nitrox_driver_name,
	.id_table = nitrox_pci_tbl,
	.probe = nitrox_probe,
	.remove	= nitrox_remove,
	.shutdown = nitrox_shutdown,
#ifdef CONFIG_PCI_IOV
	.sriov_configure = nitrox_sriov_configure,
#endif
};

module_pci_driver(nitrox_driver);

MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
MODULE_LICENSE("GPL");
MODULE_VERSION(DRIVER_VERSION);
MODULE_FIRMWARE(SE_FW);
Commit	Line	Data
14fa93cd SJ	1	#include <linux/aer.h>
14fa93cd SJ	2	#include <linux/delay.h>
086eac9e	3	#include <linux/debugfs.h>
14fa93cd SJ	4	#include <linux/firmware.h>
	5	#include <linux/list.h>
	6	#include <linux/module.h>
	7	#include <linux/mutex.h>
	8	#include <linux/pci.h>
	9	#include <linux/pci_ids.h>
	10
	11	#include "nitrox_dev.h"
	12	#include "nitrox_common.h"
	13	#include "nitrox_csr.h"
41a9aca6	14	#include "nitrox_hal.h"
14fa93cd SJ	15
	16	#define CNN55XX_DEV_ID 0x12
	17	#define MAX_PF_QUEUES 64
	18	#define UCODE_HLEN 48
	19	#define SE_GROUP 0
	20
41a9aca6	21	#define DRIVER_VERSION "1.1"
440bf347	22	#define FW_DIR "cavium/"
14fa93cd	23	/* SE microcode */
440bf347	24	#define SE_FW FW_DIR "cnn55xx_se.fw"
14fa93cd SJ	25
	26	static const char nitrox_driver_name[] = "CNN55XX";
	27
	28	static LIST_HEAD(ndevlist);
	29	static DEFINE_MUTEX(devlist_lock);
	30	static unsigned int num_devices;
	31
	32	/**
	33	* nitrox_pci_tbl - PCI Device ID Table
	34	*/
	35	static const struct pci_device_id nitrox_pci_tbl[] = {
	36	{PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
	37	/* required last entry */
	38	{0, }
	39	};
	40	MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);
	41
	42	static unsigned int qlen = DEFAULT_CMD_QLEN;
	43	module_param(qlen, uint, 0644);
	44	MODULE_PARM_DESC(qlen, "Command queue length - default 2048");
	45
41a9aca6 SJ	46	#ifdef CONFIG_PCI_IOV
	47	int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs);
	48	#else
	49	int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs)
	50	{
	51	return 0;
	52	}
	53	#endif
	54
14fa93cd SJ	55	/**
	56	* struct ucode - Firmware Header
	57	* @id: microcode ID
	58	* @version: firmware version
	59	* @code_size: code section size
	60	* @raz: alignment
	61	* @code: code section
	62	*/
	63	struct ucode {
	64	u8 id;
	65	char version[VERSION_LEN - 1];
	66	__be32 code_size;
	67	u8 raz[12];
	68	u64 code[0];
	69	};
	70
	71	/**
	72	* write_to_ucd_unit - Write Firmware to NITROX UCD unit
	73	*/
	74	static void write_to_ucd_unit(struct nitrox_device *ndev,
	75	struct ucode *ucode)
	76	{
	77	u32 code_size = be32_to_cpu(ucode->code_size) * 2;
	78	u64 offset, data;
	79	int i = 0;
	80
	81	/*
	82	* UCD structure
	83	*
	84	* -------------
	85	* \| BLK 7 \|
	86	* -------------
	87	* \| BLK 6 \|
	88	* -------------
	89	* \| ... \|
	90	* -------------
	91	* \| BLK 0 \|
	92	* -------------
	93	* Total of 8 blocks, each size 32KB
	94	*/
	95
	96	/* set the block number */
	97	offset = UCD_UCODE_LOAD_BLOCK_NUM;
	98	nitrox_write_csr(ndev, offset, 0);
	99
	100	code_size = roundup(code_size, 8);
	101	while (code_size) {
	102	data = ucode->code[i];
	103	/* write 8 bytes at a time */
	104	offset = UCD_UCODE_LOAD_IDX_DATAX(i);
	105	nitrox_write_csr(ndev, offset, data);
	106	code_size -= 8;
	107	i++;
	108	}
	109
	110	/* put all SE cores in group 0 */
	111	offset = POM_GRP_EXECMASKX(SE_GROUP);
	112	nitrox_write_csr(ndev, offset, (~0ULL));
	113
	114	for (i = 0; i < ndev->hw.se_cores; i++) {
	115	/*
	116	* write block number and firware length
	117	* bit:<2:0> block number
	118	* bit:3 is set SE uses 32KB microcode
119	* bit:3 is clear SE uses 64KB microcode
120	*/
121	offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
122	nitrox_write_csr(ndev, offset, 0x8);
123	}
124	usleep_range(300, 400);
125	}
126
127	static int nitrox_load_fw(struct nitrox_device ndev, const char fw_name)
128	{
129	const struct firmware *fw;
130	struct ucode *ucode;
131	int ret;
132
133	dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
134
135	ret = request_firmware(&fw, fw_name, DEV(ndev));
136	if (ret < 0) {
137	dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
138	return ret;
139	}
140
141	ucode = (struct ucode *)fw->data;
142	/* copy the firmware version */
143	memcpy(ndev->hw.fw_name, ucode->version, (VERSION_LEN - 2));
144	ndev->hw.fw_name[VERSION_LEN - 1] = '\0';
145
146	write_to_ucd_unit(ndev, ucode);
147	release_firmware(fw);
148
14fa93cd SJ	149	return 0;
	150	}
	151
	152	/**
	153	* nitrox_add_to_devlist - add NITROX device to global device list
	154	* @ndev: NITROX device
	155	*/
	156	static int nitrox_add_to_devlist(struct nitrox_device *ndev)
	157	{
	158	struct nitrox_device *dev;
	159	int ret = 0;
	160
	161	INIT_LIST_HEAD(&ndev->list);
	162	refcount_set(&ndev->refcnt, 1);
	163
	164	mutex_lock(&devlist_lock);
	165	list_for_each_entry(dev, &ndevlist, list) {
	166	if (dev == ndev) {
	167	ret = -EEXIST;
	168	goto unlock;
	169	}
	170	}
	171	ndev->idx = num_devices++;
	172	list_add_tail(&ndev->list, &ndevlist);
	173	unlock:
	174	mutex_unlock(&devlist_lock);
	175	return ret;
	176	}
	177
	178	/**
	179	* nitrox_remove_from_devlist - remove NITROX device from
	180	* global device list
	181	* @ndev: NITROX device
	182	*/
	183	static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
	184	{
	185	mutex_lock(&devlist_lock);
	186	list_del(&ndev->list);
	187	num_devices--;
	188	mutex_unlock(&devlist_lock);
	189	}
	190
f2663872 SJ	191	struct nitrox_device *nitrox_get_first_device(void)
	192	{
	193	struct nitrox_device *ndev = NULL;
	194
	195	mutex_lock(&devlist_lock);
	196	list_for_each_entry(ndev, &ndevlist, list) {
	197	if (nitrox_ready(ndev))
	198	break;
	199	}
	200	mutex_unlock(&devlist_lock);
	201	if (!ndev)
	202	return NULL;
	203
	204	refcount_inc(&ndev->refcnt);
	205	/* barrier to sync with other cpus */
	206	smp_mb__after_atomic();
	207	return ndev;
	208	}
	209
	210	void nitrox_put_device(struct nitrox_device *ndev)
	211	{
	212	if (!ndev)
	213	return;
	214
	215	refcount_dec(&ndev->refcnt);
	216	/* barrier to sync with other cpus */
	217	smp_mb__after_atomic();
	218	}
	219
14fa93cd SJ	220	static int nitrox_reset_device(struct pci_dev *pdev)
	221	{
	222	int pos = 0;
	223
	224	pos = pci_save_state(pdev);
	225	if (pos) {
	226	dev_err(&pdev->dev, "Failed to save pci state\n");
	227	return -ENOMEM;
	228	}
	229
	230	pos = pci_pcie_cap(pdev);
	231	if (!pos)
	232	return -ENOTTY;
	233
	234	if (!pci_wait_for_pending_transaction(pdev))
	235	dev_err(&pdev->dev, "waiting for pending transaction\n");
	236
	237	pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
	238	msleep(100);
	239	pci_restore_state(pdev);
	240
	241	return 0;
	242	}
	243
	244	static int nitrox_pf_sw_init(struct nitrox_device *ndev)
	245	{
	246	int err;
	247
	248	err = nitrox_common_sw_init(ndev);
	249	if (err)
	250	return err;
	251
	252	err = nitrox_pf_init_isr(ndev);
	253	if (err)
	254	nitrox_common_sw_cleanup(ndev);
	255
	256	return err;
	257	}
	258
	259	static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
	260	{
	261	nitrox_pf_cleanup_isr(ndev);
	262	nitrox_common_sw_cleanup(ndev);
	263	}
	264
	265	/**
	266	* nitrox_bist_check - Check NITORX BIST registers status
	267	* @ndev: NITROX device
	268	*/
	269	static int nitrox_bist_check(struct nitrox_device *ndev)
	270	{
	271	u64 value = 0;
	272	int i;
	273
	274	for (i = 0; i < NR_CLUSTERS; i++) {
	275	value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
	276	value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
	277	}
	278	value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
	279	value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
	280	value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
	281	value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
	282	value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
	283	value += nitrox_read_csr(ndev, POM_BIST_REG);
284	value += nitrox_read_csr(ndev, BMI_BIST_REG);
285	value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
286	value += nitrox_read_csr(ndev, BMO_BIST_REG);
287	value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
288	value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
289	if (value)
290	return -EIO;
291	return 0;
292	}
293
14fa93cd SJ	294	static int nitrox_pf_hw_init(struct nitrox_device *ndev)
	295	{
	296	int err;
	297
	298	err = nitrox_bist_check(ndev);
	299	if (err) {
	300	dev_err(&ndev->pdev->dev, "BIST check failed\n");
	301	return err;
	302	}
	303	/* get cores information */
	304	nitrox_get_hwinfo(ndev);
	305
	306	nitrox_config_nps_unit(ndev);
	307	nitrox_config_pom_unit(ndev);
	308	nitrox_config_efl_unit(ndev);
	309	/* configure IO units */
	310	nitrox_config_bmi_unit(ndev);
	311	nitrox_config_bmo_unit(ndev);
	312	/* configure Local Buffer Cache */
	313	nitrox_config_lbc_unit(ndev);
	314	nitrox_config_rand_unit(ndev);
	315
	316	/* load firmware on SE cores */
	317	err = nitrox_load_fw(ndev, SE_FW);
	318	if (err)
	319	return err;
	320
	321	nitrox_config_emu_unit(ndev);
	322
	323	return 0;
	324	}
	325
	326	/**
	327	* nitrox_probe - NITROX Initialization function.
	328	* @pdev: PCI device information struct
	329	* @id: entry in nitrox_pci_tbl
	330	*
	331	* Return: 0, if the driver is bound to the device, or
	332	* a negative error if there is failure.
	333	*/
	334	static int nitrox_probe(struct pci_dev *pdev,
	335	const struct pci_device_id *id)
	336	{
	337	struct nitrox_device *ndev;
	338	int err;
	339
	340	dev_info_once(&pdev->dev, "%s driver version %s\n",
	341	nitrox_driver_name, DRIVER_VERSION);
	342
	343	err = pci_enable_device_mem(pdev);
	344	if (err)
	345	return err;
	346
	347	/* do FLR */
	348	err = nitrox_reset_device(pdev);
	349	if (err) {
	350	dev_err(&pdev->dev, "FLR failed\n");
	351	pci_disable_device(pdev);
	352	return err;
	353	}
	354
	355	if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
	356	dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
	357	} else {
358	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
359	if (err) {
360	dev_err(&pdev->dev, "DMA configuration failed\n");
361	pci_disable_device(pdev);
362	return err;
363	}
364	}
365
366	err = pci_request_mem_regions(pdev, nitrox_driver_name);
367	if (err) {
368	pci_disable_device(pdev);
369	return err;
370	}
371	pci_set_master(pdev);
372
373	ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
32e67b9a CJ	374	if (!ndev) {
32e67b9a CJ	375	err = -ENOMEM;
14fa93cd	376	goto ndev_fail;
32e67b9a	377	}
14fa93cd SJ	378
	379	pci_set_drvdata(pdev, ndev);
	380	ndev->pdev = pdev;
	381
	382	/* add to device list */
	383	nitrox_add_to_devlist(ndev);
	384
	385	ndev->hw.vendor_id = pdev->vendor;
	386	ndev->hw.device_id = pdev->device;
	387	ndev->hw.revision_id = pdev->revision;
	388	/* command timeout in jiffies */
	389	ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
	390	ndev->node = dev_to_node(&pdev->dev);
	391	if (ndev->node == NUMA_NO_NODE)
	392	ndev->node = 0;
	393
	394	ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
	395	pci_resource_len(pdev, 0));
	396	if (!ndev->bar_addr) {
	397	err = -EIO;
	398	goto ioremap_err;
	399	}
	400	/* allocate command queus based on cpus, max queues are 64 */
	401	ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
	402	ndev->qlen = qlen;
	403
	404	err = nitrox_pf_sw_init(ndev);
	405	if (err)
	406	goto ioremap_err;
	407
	408	err = nitrox_pf_hw_init(ndev);
	409	if (err)
	410	goto pf_hw_fail;
	411
086eac9e SJ	412	err = nitrox_debugfs_init(ndev);
	413	if (err)
	414	goto pf_hw_fail;
	415
fec165c9 SJ	416	/* clear the statistics */
	417	atomic64_set(&ndev->stats.posted, 0);
	418	atomic64_set(&ndev->stats.completed, 0);
	419	atomic64_set(&ndev->stats.dropped, 0);
	420
41a9aca6	421	atomic_set(&ndev->state, __NDEV_READY);
14fa93cd SJ	422	/* barrier to sync with other cpus */
14fa93cd SJ	423	smp_mb__after_atomic();
f2663872 SJ	424
	425	err = nitrox_crypto_register();
	426	if (err)
	427	goto crypto_fail;
	428
14fa93cd SJ	429	return 0;
14fa93cd SJ	430
f2663872 SJ	431	crypto_fail:
f2663872 SJ	432	nitrox_debugfs_exit(ndev);
41a9aca6	433	atomic_set(&ndev->state, __NDEV_NOT_READY);
f2663872 SJ	434	/* barrier to sync with other cpus */
f2663872 SJ	435	smp_mb__after_atomic();
14fa93cd SJ	436	pf_hw_fail:
	437	nitrox_pf_sw_cleanup(ndev);
	438	ioremap_err:
	439	nitrox_remove_from_devlist(ndev);
	440	kfree(ndev);
	441	pci_set_drvdata(pdev, NULL);
	442	ndev_fail:
	443	pci_release_mem_regions(pdev);
	444	pci_disable_device(pdev);
	445	return err;
	446	}
	447
	448	/**
	449	* nitrox_remove - Unbind the driver from the device.
	450	* @pdev: PCI device information struct
	451	*/
	452	static void nitrox_remove(struct pci_dev *pdev)
	453	{
	454	struct nitrox_device *ndev = pci_get_drvdata(pdev);
	455
	456	if (!ndev)
	457	return;
	458
	459	if (!refcount_dec_and_test(&ndev->refcnt)) {
	460	dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
	461	refcount_read(&ndev->refcnt));
	462	return;
	463	}
	464
	465	dev_info(DEV(ndev), "Removing Device %x:%x\n",
	466	ndev->hw.vendor_id, ndev->hw.device_id);
	467
41a9aca6	468	atomic_set(&ndev->state, __NDEV_NOT_READY);
14fa93cd SJ	469	/* barrier to sync with other cpus */
	470	smp_mb__after_atomic();
	471
	472	nitrox_remove_from_devlist(ndev);
41a9aca6 SJ	473
	474	#ifdef CONFIG_PCI_IOV
	475	/* disable SR-IOV */
	476	nitrox_sriov_configure(pdev, 0);
	477	#endif
f2663872	478	nitrox_crypto_unregister();
086eac9e	479	nitrox_debugfs_exit(ndev);
14fa93cd SJ	480	nitrox_pf_sw_cleanup(ndev);
	481
	482	iounmap(ndev->bar_addr);
	483	kfree(ndev);
	484
	485	pci_set_drvdata(pdev, NULL);
	486	pci_release_mem_regions(pdev);
	487	pci_disable_device(pdev);
	488	}
	489
	490	static void nitrox_shutdown(struct pci_dev *pdev)
	491	{
	492	pci_set_drvdata(pdev, NULL);
	493	pci_release_mem_regions(pdev);
	494	pci_disable_device(pdev);
	495	}
	496
	497	static struct pci_driver nitrox_driver = {
	498	.name = nitrox_driver_name,
	499	.id_table = nitrox_pci_tbl,
	500	.probe = nitrox_probe,
	501	.remove = nitrox_remove,
	502	.shutdown = nitrox_shutdown,
41a9aca6 SJ	503	#ifdef CONFIG_PCI_IOV
	504	.sriov_configure = nitrox_sriov_configure,
	505	#endif
14fa93cd SJ	506	};
	507
	508	module_pci_driver(nitrox_driver);
	509
	510	MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
	511	MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
	512	MODULE_LICENSE("GPL");
	513	MODULE_VERSION(DRIVER_VERSION);
	514	MODULE_FIRMWARE(SE_FW);