[mirror_ubuntu-artful-kernel.git] / drivers / crypto / cavium / nitrox / nitrox_main.c

#include <linux/aer.h>
#include <linux/delay.h>
#include <linux/debugfs.h>
#include <linux/firmware.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/pci.h>
#include <linux/pci_ids.h>

#include "nitrox_dev.h"
#include "nitrox_common.h"
#include "nitrox_csr.h"

#define CNN55XX_DEV_ID	0x12
#define MAX_PF_QUEUES	64
#define UCODE_HLEN 48
#define SE_GROUP 0

#define DRIVER_VERSION "1.0"
/* SE microcode */
#define SE_FW	"cnn55xx_se.fw"

static const char nitrox_driver_name[] = "CNN55XX";

static LIST_HEAD(ndevlist);
static DEFINE_MUTEX(devlist_lock);
static unsigned int num_devices;

/**
 * nitrox_pci_tbl - PCI Device ID Table
 */
static const struct pci_device_id nitrox_pci_tbl[] = {
	{PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
	/* required last entry */
	{0, }
};
MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);

static unsigned int qlen = DEFAULT_CMD_QLEN;
module_param(qlen, uint, 0644);
MODULE_PARM_DESC(qlen, "Command queue length - default 2048");

/**
 * struct ucode - Firmware Header
 * @id: microcode ID
 * @version: firmware version
 * @code_size: code section size
 * @raz: alignment
 * @code: code section
 */
struct ucode {
	u8 id;
	char version[VERSION_LEN - 1];
	__be32 code_size;
	u8 raz[12];
	u64 code[0];
};

/**
 * write_to_ucd_unit - Write Firmware to NITROX UCD unit
 */
static void write_to_ucd_unit(struct nitrox_device *ndev,
			      struct ucode *ucode)
{
	u32 code_size = be32_to_cpu(ucode->code_size) * 2;
	u64 offset, data;
	int i = 0;

	/*
	 * UCD structure
	 *
	 *  -------------
	 *  |    BLK 7  |
	 *  -------------
	 *  |    BLK 6  |
	 *  -------------
	 *  |    ...    |
	 *  -------------
	 *  |    BLK 0  |
	 *  -------------
	 *  Total of 8 blocks, each size 32KB
	 */

	/* set the block number */
	offset = UCD_UCODE_LOAD_BLOCK_NUM;
	nitrox_write_csr(ndev, offset, 0);

	code_size = roundup(code_size, 8);
	while (code_size) {
		data = ucode->code[i];
		/* write 8 bytes at a time */
		offset = UCD_UCODE_LOAD_IDX_DATAX(i);
		nitrox_write_csr(ndev, offset, data);
		code_size -= 8;
		i++;
	}

	/* put all SE cores in group 0 */
	offset = POM_GRP_EXECMASKX(SE_GROUP);
	nitrox_write_csr(ndev, offset, (~0ULL));

	for (i = 0; i < ndev->hw.se_cores; i++) {
		/*
		 * write block number and firware length
		 * bit:<2:0> block number
		 * bit:3 is set SE uses 32KB microcode
		 * bit:3 is clear SE uses 64KB microcode
		 */
		offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
		nitrox_write_csr(ndev, offset, 0x8);
	}
	usleep_range(300, 400);
}

static int nitrox_load_fw(struct nitrox_device *ndev, const char *fw_name)
{
	const struct firmware *fw;
	struct ucode *ucode;
	int ret;

	dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);

	ret = request_firmware(&fw, fw_name, DEV(ndev));
	if (ret < 0) {
		dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
		return ret;
	}

	ucode = (struct ucode *)fw->data;
	/* copy the firmware version */
	memcpy(ndev->hw.fw_name, ucode->version, (VERSION_LEN - 2));
	ndev->hw.fw_name[VERSION_LEN - 1] = '\0';

	write_to_ucd_unit(ndev, ucode);
	release_firmware(fw);

	set_bit(NITROX_UCODE_LOADED, &ndev->status);
	/* barrier to sync with other cpus */
	smp_mb__after_atomic();
	return 0;
}

/**
 * nitrox_add_to_devlist - add NITROX device to global device list
 * @ndev: NITROX device
 */
static int nitrox_add_to_devlist(struct nitrox_device *ndev)
{
	struct nitrox_device *dev;
	int ret = 0;

	INIT_LIST_HEAD(&ndev->list);
	refcount_set(&ndev->refcnt, 1);

	mutex_lock(&devlist_lock);
	list_for_each_entry(dev, &ndevlist, list) {
		if (dev == ndev) {
			ret = -EEXIST;
			goto unlock;
		}
	}
	ndev->idx = num_devices++;
	list_add_tail(&ndev->list, &ndevlist);
unlock:
	mutex_unlock(&devlist_lock);
	return ret;
}

/**
 * nitrox_remove_from_devlist - remove NITROX device from
 *   global device list
 * @ndev: NITROX device
 */
static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
{
	mutex_lock(&devlist_lock);
	list_del(&ndev->list);
	num_devices--;
	mutex_unlock(&devlist_lock);
}

struct nitrox_device *nitrox_get_first_device(void)
{
	struct nitrox_device *ndev = NULL;

	mutex_lock(&devlist_lock);
	list_for_each_entry(ndev, &ndevlist, list) {
		if (nitrox_ready(ndev))
			break;
	}
	mutex_unlock(&devlist_lock);
	if (!ndev)
		return NULL;

	refcount_inc(&ndev->refcnt);
	/* barrier to sync with other cpus */
	smp_mb__after_atomic();
	return ndev;
}

void nitrox_put_device(struct nitrox_device *ndev)
{
	if (!ndev)
		return;

	refcount_dec(&ndev->refcnt);
	/* barrier to sync with other cpus */
	smp_mb__after_atomic();
}

static int nitrox_reset_device(struct pci_dev *pdev)
{
	int pos = 0;

	pos = pci_save_state(pdev);
	if (pos) {
		dev_err(&pdev->dev, "Failed to save pci state\n");
		return -ENOMEM;
	}

	pos = pci_pcie_cap(pdev);
	if (!pos)
		return -ENOTTY;

	if (!pci_wait_for_pending_transaction(pdev))
		dev_err(&pdev->dev, "waiting for pending transaction\n");

	pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
	msleep(100);
	pci_restore_state(pdev);

	return 0;
}

static int nitrox_pf_sw_init(struct nitrox_device *ndev)
{
	int err;

	err = nitrox_common_sw_init(ndev);
	if (err)
		return err;

	err = nitrox_pf_init_isr(ndev);
	if (err)
		nitrox_common_sw_cleanup(ndev);

	return err;
}

static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
{
	nitrox_pf_cleanup_isr(ndev);
	nitrox_common_sw_cleanup(ndev);
}

/**
 * nitrox_bist_check - Check NITORX BIST registers status
 * @ndev: NITROX device
 */
static int nitrox_bist_check(struct nitrox_device *ndev)
{
	u64 value = 0;
	int i;

	for (i = 0; i < NR_CLUSTERS; i++) {
		value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
		value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
	}
	value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
	value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
	value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
	value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
	value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
	value += nitrox_read_csr(ndev, POM_BIST_REG);
	value += nitrox_read_csr(ndev, BMI_BIST_REG);
	value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
	value += nitrox_read_csr(ndev, BMO_BIST_REG);
	value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
	value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
	if (value)
		return -EIO;
	return 0;
}

static void nitrox_get_hwinfo(struct nitrox_device *ndev)
{
	union emu_fuse_map emu_fuse;
	u64 offset;
	int i;

	for (i = 0; i < NR_CLUSTERS; i++) {
		u8 dead_cores;

		offset = EMU_FUSE_MAPX(i);
		emu_fuse.value = nitrox_read_csr(ndev, offset);
		if (emu_fuse.s.valid) {
			dead_cores = hweight32(emu_fuse.s.ae_fuse);
			ndev->hw.ae_cores += AE_CORES_PER_CLUSTER - dead_cores;
			dead_cores = hweight16(emu_fuse.s.se_fuse);
			ndev->hw.se_cores += SE_CORES_PER_CLUSTER - dead_cores;
		}
	}
}

static int nitrox_pf_hw_init(struct nitrox_device *ndev)
{
	int err;

	err = nitrox_bist_check(ndev);
	if (err) {
		dev_err(&ndev->pdev->dev, "BIST check failed\n");
		return err;
	}
	/* get cores information */
	nitrox_get_hwinfo(ndev);

	nitrox_config_nps_unit(ndev);
	nitrox_config_pom_unit(ndev);
	nitrox_config_efl_unit(ndev);
	/* configure IO units */
	nitrox_config_bmi_unit(ndev);
	nitrox_config_bmo_unit(ndev);
	/* configure Local Buffer Cache */
	nitrox_config_lbc_unit(ndev);
	nitrox_config_rand_unit(ndev);

	/* load firmware on SE cores */
	err = nitrox_load_fw(ndev, SE_FW);
	if (err)
		return err;

	nitrox_config_emu_unit(ndev);

	return 0;
}

#if IS_ENABLED(CONFIG_DEBUG_FS)
static int registers_show(struct seq_file *s, void *v)
{
	struct nitrox_device *ndev = s->private;
	u64 offset;

	/* NPS DMA stats */
	offset = NPS_STATS_PKT_DMA_RD_CNT;
	seq_printf(s, "NPS_STATS_PKT_DMA_RD_CNT  0x%016llx\n",
		   nitrox_read_csr(ndev, offset));
	offset = NPS_STATS_PKT_DMA_WR_CNT;
	seq_printf(s, "NPS_STATS_PKT_DMA_WR_CNT  0x%016llx\n",
		   nitrox_read_csr(ndev, offset));

	/* BMI/BMO stats */
	offset = BMI_NPS_PKT_CNT;
	seq_printf(s, "BMI_NPS_PKT_CNT  0x%016llx\n",
		   nitrox_read_csr(ndev, offset));
	offset = BMO_NPS_SLC_PKT_CNT;
	seq_printf(s, "BMO_NPS_PKT_CNT  0x%016llx\n",
		   nitrox_read_csr(ndev, offset));

	return 0;
}

static int registers_open(struct inode *inode, struct file *file)
{
	return single_open(file, registers_show, inode->i_private);
}

static const struct file_operations register_fops = {
	.owner = THIS_MODULE,
	.open = registers_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = single_release,
};

static int firmware_show(struct seq_file *s, void *v)
{
	struct nitrox_device *ndev = s->private;

	seq_printf(s, "Version: %s\n", ndev->hw.fw_name);
	return 0;
}

static int firmware_open(struct inode *inode, struct file *file)
{
	return single_open(file, firmware_show, inode->i_private);
}

static const struct file_operations firmware_fops = {
	.owner = THIS_MODULE,
	.open = firmware_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = single_release,
};

static int nitrox_show(struct seq_file *s, void *v)
{
	struct nitrox_device *ndev = s->private;

	seq_printf(s, "NITROX-5 [idx: %d]\n", ndev->idx);
	seq_printf(s, "  Revsion ID: 0x%0x\n", ndev->hw.revision_id);
	seq_printf(s, "  Cores [AE: %u  SE: %u]\n",
		   ndev->hw.ae_cores, ndev->hw.se_cores);
	seq_printf(s, "  Number of Queues: %u\n", ndev->nr_queues);
	seq_printf(s, "  Queue length: %u\n", ndev->qlen);
	seq_printf(s, "  Node: %u\n", ndev->node);

	return 0;
}

static int nitrox_open(struct inode *inode, struct file *file)
{
	return single_open(file, nitrox_show, inode->i_private);
}

static const struct file_operations nitrox_fops = {
	.owner = THIS_MODULE,
	.open = nitrox_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = single_release,
};

static void nitrox_debugfs_exit(struct nitrox_device *ndev)
{
	debugfs_remove_recursive(ndev->debugfs_dir);
	ndev->debugfs_dir = NULL;
}

static int nitrox_debugfs_init(struct nitrox_device *ndev)
{
	struct dentry *dir, *f;

	dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
	if (!dir)
		return -ENOMEM;

	ndev->debugfs_dir = dir;
	f = debugfs_create_file("counters", 0400, dir, ndev, &register_fops);
	if (!f)
		goto err;
	f = debugfs_create_file("firmware", 0400, dir, ndev, &firmware_fops);
	if (!f)
		goto err;
	f = debugfs_create_file("nitrox", 0400, dir, ndev, &nitrox_fops);
	if (!f)
		goto err;

	return 0;

err:
	nitrox_debugfs_exit(ndev);
	return -ENODEV;
}
#else
static int nitrox_debugfs_init(struct nitrox_device *ndev)
{
	return 0;
}

static void nitrox_debugfs_exit(struct nitrox_device *ndev)
{
}
#endif

/**
 * nitrox_probe - NITROX Initialization function.
 * @pdev: PCI device information struct
 * @id: entry in nitrox_pci_tbl
 *
 * Return: 0, if the driver is bound to the device, or
 *         a negative error if there is failure.
 */
static int nitrox_probe(struct pci_dev *pdev,
			const struct pci_device_id *id)
{
	struct nitrox_device *ndev;
	int err;

	dev_info_once(&pdev->dev, "%s driver version %s\n",
		      nitrox_driver_name, DRIVER_VERSION);

	err = pci_enable_device_mem(pdev);
	if (err)
		return err;

	/* do FLR */
	err = nitrox_reset_device(pdev);
	if (err) {
		dev_err(&pdev->dev, "FLR failed\n");
		pci_disable_device(pdev);
		return err;
	}

	if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
		dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
	} else {
		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
		if (err) {
			dev_err(&pdev->dev, "DMA configuration failed\n");
			pci_disable_device(pdev);
			return err;
		}
	}

	err = pci_request_mem_regions(pdev, nitrox_driver_name);
	if (err) {
		pci_disable_device(pdev);
		return err;
	}
	pci_set_master(pdev);

	ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
	if (!ndev)
		goto ndev_fail;

	pci_set_drvdata(pdev, ndev);
	ndev->pdev = pdev;

	/* add to device list */
	nitrox_add_to_devlist(ndev);

	ndev->hw.vendor_id = pdev->vendor;
	ndev->hw.device_id = pdev->device;
	ndev->hw.revision_id = pdev->revision;
	/* command timeout in jiffies */
	ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
	ndev->node = dev_to_node(&pdev->dev);
	if (ndev->node == NUMA_NO_NODE)
		ndev->node = 0;

	ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
				 pci_resource_len(pdev, 0));
	if (!ndev->bar_addr) {
		err = -EIO;
		goto ioremap_err;
	}
	/* allocate command queus based on cpus, max queues are 64 */
	ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
	ndev->qlen = qlen;

	err = nitrox_pf_sw_init(ndev);
	if (err)
		goto ioremap_err;

	err = nitrox_pf_hw_init(ndev);
	if (err)
		goto pf_hw_fail;

	err = nitrox_debugfs_init(ndev);
	if (err)
		goto pf_hw_fail;

	set_bit(NITROX_READY, &ndev->status);
	/* barrier to sync with other cpus */
	smp_mb__after_atomic();

	err = nitrox_crypto_register();
	if (err)
		goto crypto_fail;

	return 0;

crypto_fail:
	nitrox_debugfs_exit(ndev);
	clear_bit(NITROX_READY, &ndev->status);
	/* barrier to sync with other cpus */
	smp_mb__after_atomic();
pf_hw_fail:
	nitrox_pf_sw_cleanup(ndev);
ioremap_err:
	nitrox_remove_from_devlist(ndev);
	kfree(ndev);
	pci_set_drvdata(pdev, NULL);
ndev_fail:
	pci_release_mem_regions(pdev);
	pci_disable_device(pdev);
	return err;
}

/**
 * nitrox_remove - Unbind the driver from the device.
 * @pdev: PCI device information struct
 */
static void nitrox_remove(struct pci_dev *pdev)
{
	struct nitrox_device *ndev = pci_get_drvdata(pdev);

	if (!ndev)
		return;

	if (!refcount_dec_and_test(&ndev->refcnt)) {
		dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
			refcount_read(&ndev->refcnt));
		return;
	}

	dev_info(DEV(ndev), "Removing Device %x:%x\n",
		 ndev->hw.vendor_id, ndev->hw.device_id);

	clear_bit(NITROX_READY, &ndev->status);
	/* barrier to sync with other cpus */
	smp_mb__after_atomic();

	nitrox_remove_from_devlist(ndev);
	nitrox_crypto_unregister();
	nitrox_debugfs_exit(ndev);
	nitrox_pf_sw_cleanup(ndev);

	iounmap(ndev->bar_addr);
	kfree(ndev);

	pci_set_drvdata(pdev, NULL);
	pci_release_mem_regions(pdev);
	pci_disable_device(pdev);
}

static void nitrox_shutdown(struct pci_dev *pdev)
{
	pci_set_drvdata(pdev, NULL);
	pci_release_mem_regions(pdev);
	pci_disable_device(pdev);
}

static struct pci_driver nitrox_driver = {
	.name = nitrox_driver_name,
	.id_table = nitrox_pci_tbl,
	.probe = nitrox_probe,
	.remove	= nitrox_remove,
	.shutdown = nitrox_shutdown,
};

module_pci_driver(nitrox_driver);

MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
MODULE_LICENSE("GPL");
MODULE_VERSION(DRIVER_VERSION);
MODULE_FIRMWARE(SE_FW);
Commit	Line	Data
14fa93cd SJ	1	#include <linux/aer.h>
14fa93cd SJ	2	#include <linux/delay.h>
086eac9e	3	#include <linux/debugfs.h>
14fa93cd SJ	4	#include <linux/firmware.h>
	5	#include <linux/list.h>
	6	#include <linux/module.h>
	7	#include <linux/mutex.h>
	8	#include <linux/pci.h>
	9	#include <linux/pci_ids.h>
	10
	11	#include "nitrox_dev.h"
	12	#include "nitrox_common.h"
	13	#include "nitrox_csr.h"
	14
	15	#define CNN55XX_DEV_ID 0x12
	16	#define MAX_PF_QUEUES 64
	17	#define UCODE_HLEN 48
	18	#define SE_GROUP 0
	19
	20	#define DRIVER_VERSION "1.0"
	21	/* SE microcode */
	22	#define SE_FW "cnn55xx_se.fw"
	23
	24	static const char nitrox_driver_name[] = "CNN55XX";
	25
	26	static LIST_HEAD(ndevlist);
	27	static DEFINE_MUTEX(devlist_lock);
	28	static unsigned int num_devices;
	29
	30	/**
	31	* nitrox_pci_tbl - PCI Device ID Table
	32	*/
	33	static const struct pci_device_id nitrox_pci_tbl[] = {
	34	{PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
	35	/* required last entry */
	36	{0, }
	37	};
	38	MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);
	39
	40	static unsigned int qlen = DEFAULT_CMD_QLEN;
	41	module_param(qlen, uint, 0644);
	42	MODULE_PARM_DESC(qlen, "Command queue length - default 2048");
	43
	44	/**
	45	* struct ucode - Firmware Header
	46	* @id: microcode ID
	47	* @version: firmware version
	48	* @code_size: code section size
	49	* @raz: alignment
	50	* @code: code section
	51	*/
	52	struct ucode {
	53	u8 id;
	54	char version[VERSION_LEN - 1];
	55	__be32 code_size;
	56	u8 raz[12];
	57	u64 code[0];
	58	};
	59
	60	/**
	61	* write_to_ucd_unit - Write Firmware to NITROX UCD unit
	62	*/
	63	static void write_to_ucd_unit(struct nitrox_device *ndev,
	64	struct ucode *ucode)
	65	{
	66	u32 code_size = be32_to_cpu(ucode->code_size) * 2;
	67	u64 offset, data;
68	int i = 0;
69
70	/*
71	* UCD structure
72	*
73	* -------------
74	* \| BLK 7 \|
75	* -------------
76	* \| BLK 6 \|
77	* -------------
78	* \| ... \|
79	* -------------
80	* \| BLK 0 \|
81	* -------------
82	* Total of 8 blocks, each size 32KB
83	*/
84
85	/* set the block number */
86	offset = UCD_UCODE_LOAD_BLOCK_NUM;
87	nitrox_write_csr(ndev, offset, 0);
88
89	code_size = roundup(code_size, 8);
90	while (code_size) {
91	data = ucode->code[i];
92	/* write 8 bytes at a time */
93	offset = UCD_UCODE_LOAD_IDX_DATAX(i);
94	nitrox_write_csr(ndev, offset, data);
95	code_size -= 8;
96	i++;
97	}
98
99	/* put all SE cores in group 0 */
100	offset = POM_GRP_EXECMASKX(SE_GROUP);
101	nitrox_write_csr(ndev, offset, (~0ULL));
102
103	for (i = 0; i < ndev->hw.se_cores; i++) {
104	/*
105	* write block number and firware length
106	* bit:<2:0> block number
107	* bit:3 is set SE uses 32KB microcode
108	* bit:3 is clear SE uses 64KB microcode
109	*/
110	offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
111	nitrox_write_csr(ndev, offset, 0x8);
112	}
113	usleep_range(300, 400);
114	}
115
116	static int nitrox_load_fw(struct nitrox_device ndev, const char fw_name)
117	{
118	const struct firmware *fw;
119	struct ucode *ucode;
120	int ret;
121
122	dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
123
124	ret = request_firmware(&fw, fw_name, DEV(ndev));
125	if (ret < 0) {
126	dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
127	return ret;
128	}
129
130	ucode = (struct ucode *)fw->data;
131	/* copy the firmware version */
132	memcpy(ndev->hw.fw_name, ucode->version, (VERSION_LEN - 2));
133	ndev->hw.fw_name[VERSION_LEN - 1] = '\0';
134
135	write_to_ucd_unit(ndev, ucode);
136	release_firmware(fw);
137
138	set_bit(NITROX_UCODE_LOADED, &ndev->status);
139	/* barrier to sync with other cpus */
140	smp_mb__after_atomic();
141	return 0;
142	}
143
144	/**
145	* nitrox_add_to_devlist - add NITROX device to global device list
146	* @ndev: NITROX device
147	*/
148	static int nitrox_add_to_devlist(struct nitrox_device *ndev)
149	{
150	struct nitrox_device *dev;
151	int ret = 0;
152
153	INIT_LIST_HEAD(&ndev->list);
154	refcount_set(&ndev->refcnt, 1);
155
156	mutex_lock(&devlist_lock);
157	list_for_each_entry(dev, &ndevlist, list) {
158	if (dev == ndev) {
159	ret = -EEXIST;
160	goto unlock;
161	}
162	}
163	ndev->idx = num_devices++;
164	list_add_tail(&ndev->list, &ndevlist);
165	unlock:
166	mutex_unlock(&devlist_lock);
167	return ret;
168	}
169
170	/**
171	* nitrox_remove_from_devlist - remove NITROX device from
172	* global device list
173	* @ndev: NITROX device
174	*/
175	static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
176	{
177	mutex_lock(&devlist_lock);
178	list_del(&ndev->list);
179	num_devices--;
180	mutex_unlock(&devlist_lock);
181	}
182
f2663872 SJ	183	struct nitrox_device *nitrox_get_first_device(void)
	184	{
	185	struct nitrox_device *ndev = NULL;
	186
	187	mutex_lock(&devlist_lock);
	188	list_for_each_entry(ndev, &ndevlist, list) {
	189	if (nitrox_ready(ndev))
	190	break;
	191	}
	192	mutex_unlock(&devlist_lock);
	193	if (!ndev)
	194	return NULL;
	195
	196	refcount_inc(&ndev->refcnt);
	197	/* barrier to sync with other cpus */
	198	smp_mb__after_atomic();
	199	return ndev;
	200	}
	201
	202	void nitrox_put_device(struct nitrox_device *ndev)
	203	{
	204	if (!ndev)
	205	return;
	206
	207	refcount_dec(&ndev->refcnt);
	208	/* barrier to sync with other cpus */
	209	smp_mb__after_atomic();
	210	}
	211
14fa93cd SJ	212	static int nitrox_reset_device(struct pci_dev *pdev)
	213	{
	214	int pos = 0;
	215
	216	pos = pci_save_state(pdev);
	217	if (pos) {
	218	dev_err(&pdev->dev, "Failed to save pci state\n");
	219	return -ENOMEM;
	220	}
	221
	222	pos = pci_pcie_cap(pdev);
	223	if (!pos)
	224	return -ENOTTY;
	225
	226	if (!pci_wait_for_pending_transaction(pdev))
	227	dev_err(&pdev->dev, "waiting for pending transaction\n");
	228
	229	pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
	230	msleep(100);
	231	pci_restore_state(pdev);
	232
	233	return 0;
	234	}
	235
	236	static int nitrox_pf_sw_init(struct nitrox_device *ndev)
	237	{
	238	int err;
	239
	240	err = nitrox_common_sw_init(ndev);
	241	if (err)
	242	return err;
	243
	244	err = nitrox_pf_init_isr(ndev);
	245	if (err)
	246	nitrox_common_sw_cleanup(ndev);
	247
	248	return err;
	249	}
	250
	251	static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
	252	{
	253	nitrox_pf_cleanup_isr(ndev);
	254	nitrox_common_sw_cleanup(ndev);
	255	}
	256
	257	/**
	258	* nitrox_bist_check - Check NITORX BIST registers status
	259	* @ndev: NITROX device
	260	*/
	261	static int nitrox_bist_check(struct nitrox_device *ndev)
	262	{
	263	u64 value = 0;
	264	int i;
	265
	266	for (i = 0; i < NR_CLUSTERS; i++) {
	267	value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
	268	value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
	269	}
	270	value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
	271	value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
	272	value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
	273	value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
	274	value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
	275	value += nitrox_read_csr(ndev, POM_BIST_REG);
276	value += nitrox_read_csr(ndev, BMI_BIST_REG);
277	value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
278	value += nitrox_read_csr(ndev, BMO_BIST_REG);
279	value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
280	value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
281	if (value)
282	return -EIO;
283	return 0;
284	}
285
286	static void nitrox_get_hwinfo(struct nitrox_device *ndev)
287	{
288	union emu_fuse_map emu_fuse;
289	u64 offset;
290	int i;
291
292	for (i = 0; i < NR_CLUSTERS; i++) {
293	u8 dead_cores;
294
295	offset = EMU_FUSE_MAPX(i);
296	emu_fuse.value = nitrox_read_csr(ndev, offset);
297	if (emu_fuse.s.valid) {
298	dead_cores = hweight32(emu_fuse.s.ae_fuse);
299	ndev->hw.ae_cores += AE_CORES_PER_CLUSTER - dead_cores;
300	dead_cores = hweight16(emu_fuse.s.se_fuse);
301	ndev->hw.se_cores += SE_CORES_PER_CLUSTER - dead_cores;
302	}
303	}
304	}
305
306	static int nitrox_pf_hw_init(struct nitrox_device *ndev)
307	{
308	int err;
309
310	err = nitrox_bist_check(ndev);
311	if (err) {
312	dev_err(&ndev->pdev->dev, "BIST check failed\n");
313	return err;
314	}
315	/* get cores information */
316	nitrox_get_hwinfo(ndev);
317
318	nitrox_config_nps_unit(ndev);
319	nitrox_config_pom_unit(ndev);
320	nitrox_config_efl_unit(ndev);
321	/* configure IO units */
322	nitrox_config_bmi_unit(ndev);
323	nitrox_config_bmo_unit(ndev);
324	/* configure Local Buffer Cache */
325	nitrox_config_lbc_unit(ndev);
326	nitrox_config_rand_unit(ndev);
327
328	/* load firmware on SE cores */
329	err = nitrox_load_fw(ndev, SE_FW);
330	if (err)
331	return err;
332
333	nitrox_config_emu_unit(ndev);
334
335	return 0;
336	}
337
086eac9e SJ	338	#if IS_ENABLED(CONFIG_DEBUG_FS)
	339	static int registers_show(struct seq_file s, void v)
	340	{
	341	struct nitrox_device *ndev = s->private;
	342	u64 offset;
	343
	344	/* NPS DMA stats */
	345	offset = NPS_STATS_PKT_DMA_RD_CNT;
	346	seq_printf(s, "NPS_STATS_PKT_DMA_RD_CNT 0x%016llx\n",
	347	nitrox_read_csr(ndev, offset));
	348	offset = NPS_STATS_PKT_DMA_WR_CNT;
	349	seq_printf(s, "NPS_STATS_PKT_DMA_WR_CNT 0x%016llx\n",
	350	nitrox_read_csr(ndev, offset));
	351
	352	/* BMI/BMO stats */
	353	offset = BMI_NPS_PKT_CNT;
	354	seq_printf(s, "BMI_NPS_PKT_CNT 0x%016llx\n",
	355	nitrox_read_csr(ndev, offset));
	356	offset = BMO_NPS_SLC_PKT_CNT;
	357	seq_printf(s, "BMO_NPS_PKT_CNT 0x%016llx\n",
	358	nitrox_read_csr(ndev, offset));
	359
	360	return 0;
	361	}
	362
	363	static int registers_open(struct inode inode, struct file file)
	364	{
	365	return single_open(file, registers_show, inode->i_private);
	366	}
	367
	368	static const struct file_operations register_fops = {
	369	.owner = THIS_MODULE,
	370	.open = registers_open,
	371	.read = seq_read,
	372	.llseek = seq_lseek,
	373	.release = single_release,
	374	};
	375
	376	static int firmware_show(struct seq_file s, void v)
	377	{
	378	struct nitrox_device *ndev = s->private;
	379
	380	seq_printf(s, "Version: %s\n", ndev->hw.fw_name);
	381	return 0;
	382	}
	383
	384	static int firmware_open(struct inode inode, struct file file)
	385	{
	386	return single_open(file, firmware_show, inode->i_private);
	387	}
	388
	389	static const struct file_operations firmware_fops = {
	390	.owner = THIS_MODULE,
	391	.open = firmware_open,
	392	.read = seq_read,
	393	.llseek = seq_lseek,
	394	.release = single_release,
	395	};
	396
	397	static int nitrox_show(struct seq_file s, void v)
	398	{
	399	struct nitrox_device *ndev = s->private;
	400
	401	seq_printf(s, "NITROX-5 [idx: %d]\n", ndev->idx);
402	seq_printf(s, " Revsion ID: 0x%0x\n", ndev->hw.revision_id);
403	seq_printf(s, " Cores [AE: %u SE: %u]\n",
404	ndev->hw.ae_cores, ndev->hw.se_cores);
405	seq_printf(s, " Number of Queues: %u\n", ndev->nr_queues);
406	seq_printf(s, " Queue length: %u\n", ndev->qlen);
407	seq_printf(s, " Node: %u\n", ndev->node);
408
409	return 0;
410	}
411
412	static int nitrox_open(struct inode inode, struct file file)
413	{
414	return single_open(file, nitrox_show, inode->i_private);
415	}
416
417	static const struct file_operations nitrox_fops = {
418	.owner = THIS_MODULE,
419	.open = nitrox_open,
420	.read = seq_read,
421	.llseek = seq_lseek,
422	.release = single_release,
423	};
424
425	static void nitrox_debugfs_exit(struct nitrox_device *ndev)
426	{
427	debugfs_remove_recursive(ndev->debugfs_dir);
428	ndev->debugfs_dir = NULL;
429	}
430
431	static int nitrox_debugfs_init(struct nitrox_device *ndev)
432	{
433	struct dentry dir, f;
434
435	dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
436	if (!dir)
437	return -ENOMEM;
438
439	ndev->debugfs_dir = dir;
440	f = debugfs_create_file("counters", 0400, dir, ndev, &register_fops);
441	if (!f)
442	goto err;
443	f = debugfs_create_file("firmware", 0400, dir, ndev, &firmware_fops);
444	if (!f)
445	goto err;
446	f = debugfs_create_file("nitrox", 0400, dir, ndev, &nitrox_fops);
447	if (!f)
448	goto err;
449
450	return 0;
451
452	err:
453	nitrox_debugfs_exit(ndev);
454	return -ENODEV;
455	}
456	#else
457	static int nitrox_debugfs_init(struct nitrox_device *ndev)
458	{
459	return 0;
460	}
461
462	static void nitrox_debugfs_exit(struct nitrox_device *ndev)
463	{
464	}
465	#endif
466
14fa93cd SJ	467	/**
	468	* nitrox_probe - NITROX Initialization function.
	469	* @pdev: PCI device information struct
	470	* @id: entry in nitrox_pci_tbl
	471	*
	472	* Return: 0, if the driver is bound to the device, or
	473	* a negative error if there is failure.
	474	*/
	475	static int nitrox_probe(struct pci_dev *pdev,
	476	const struct pci_device_id *id)
	477	{
	478	struct nitrox_device *ndev;
	479	int err;
	480
	481	dev_info_once(&pdev->dev, "%s driver version %s\n",
	482	nitrox_driver_name, DRIVER_VERSION);
	483
	484	err = pci_enable_device_mem(pdev);
	485	if (err)
	486	return err;
	487
	488	/* do FLR */
	489	err = nitrox_reset_device(pdev);
	490	if (err) {
	491	dev_err(&pdev->dev, "FLR failed\n");
	492	pci_disable_device(pdev);
	493	return err;
	494	}
	495
	496	if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
	497	dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
	498	} else {
	499	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
	500	if (err) {
	501	dev_err(&pdev->dev, "DMA configuration failed\n");
	502	pci_disable_device(pdev);
	503	return err;
	504	}
	505	}
	506
	507	err = pci_request_mem_regions(pdev, nitrox_driver_name);
	508	if (err) {
	509	pci_disable_device(pdev);
	510	return err;
	511	}
	512	pci_set_master(pdev);
	513
	514	ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
	515	if (!ndev)
	516	goto ndev_fail;
	517
	518	pci_set_drvdata(pdev, ndev);
	519	ndev->pdev = pdev;
	520
	521	/* add to device list */
	522	nitrox_add_to_devlist(ndev);
	523
	524	ndev->hw.vendor_id = pdev->vendor;
	525	ndev->hw.device_id = pdev->device;
	526	ndev->hw.revision_id = pdev->revision;
	527	/* command timeout in jiffies */
	528	ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
	529	ndev->node = dev_to_node(&pdev->dev);
	530	if (ndev->node == NUMA_NO_NODE)
531	ndev->node = 0;
532
533	ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
534	pci_resource_len(pdev, 0));
535	if (!ndev->bar_addr) {
536	err = -EIO;
537	goto ioremap_err;
538	}
539	/* allocate command queus based on cpus, max queues are 64 */
540	ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
541	ndev->qlen = qlen;
542
543	err = nitrox_pf_sw_init(ndev);
544	if (err)
545	goto ioremap_err;
546
547	err = nitrox_pf_hw_init(ndev);
548	if (err)
549	goto pf_hw_fail;
550
086eac9e SJ	551	err = nitrox_debugfs_init(ndev);
	552	if (err)
	553	goto pf_hw_fail;
	554
14fa93cd SJ	555	set_bit(NITROX_READY, &ndev->status);
	556	/* barrier to sync with other cpus */
	557	smp_mb__after_atomic();
f2663872 SJ	558
	559	err = nitrox_crypto_register();
	560	if (err)
	561	goto crypto_fail;
	562
14fa93cd SJ	563	return 0;
14fa93cd SJ	564
f2663872 SJ	565	crypto_fail:
	566	nitrox_debugfs_exit(ndev);
	567	clear_bit(NITROX_READY, &ndev->status);
	568	/* barrier to sync with other cpus */
	569	smp_mb__after_atomic();
14fa93cd SJ	570	pf_hw_fail:
	571	nitrox_pf_sw_cleanup(ndev);
	572	ioremap_err:
	573	nitrox_remove_from_devlist(ndev);
	574	kfree(ndev);
	575	pci_set_drvdata(pdev, NULL);
	576	ndev_fail:
	577	pci_release_mem_regions(pdev);
	578	pci_disable_device(pdev);
	579	return err;
	580	}
	581
	582	/**
	583	* nitrox_remove - Unbind the driver from the device.
	584	* @pdev: PCI device information struct
	585	*/
	586	static void nitrox_remove(struct pci_dev *pdev)
	587	{
	588	struct nitrox_device *ndev = pci_get_drvdata(pdev);
	589
	590	if (!ndev)
	591	return;
	592
	593	if (!refcount_dec_and_test(&ndev->refcnt)) {
	594	dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
	595	refcount_read(&ndev->refcnt));
	596	return;
	597	}
	598
	599	dev_info(DEV(ndev), "Removing Device %x:%x\n",
	600	ndev->hw.vendor_id, ndev->hw.device_id);
	601
	602	clear_bit(NITROX_READY, &ndev->status);
	603	/* barrier to sync with other cpus */
	604	smp_mb__after_atomic();
	605
	606	nitrox_remove_from_devlist(ndev);
f2663872	607	nitrox_crypto_unregister();
086eac9e	608	nitrox_debugfs_exit(ndev);
14fa93cd SJ	609	nitrox_pf_sw_cleanup(ndev);
	610
	611	iounmap(ndev->bar_addr);
	612	kfree(ndev);
	613
	614	pci_set_drvdata(pdev, NULL);
	615	pci_release_mem_regions(pdev);
	616	pci_disable_device(pdev);
	617	}
	618
	619	static void nitrox_shutdown(struct pci_dev *pdev)
	620	{
	621	pci_set_drvdata(pdev, NULL);
	622	pci_release_mem_regions(pdev);
	623	pci_disable_device(pdev);
	624	}
	625
	626	static struct pci_driver nitrox_driver = {
	627	.name = nitrox_driver_name,
	628	.id_table = nitrox_pci_tbl,
	629	.probe = nitrox_probe,
	630	.remove = nitrox_remove,
	631	.shutdown = nitrox_shutdown,
	632	};
	633
	634	module_pci_driver(nitrox_driver);
	635
	636	MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
	637	MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
	638	MODULE_LICENSE("GPL");
	639	MODULE_VERSION(DRIVER_VERSION);
	640	MODULE_FIRMWARE(SE_FW);