[mirror_ubuntu-artful-kernel.git] / drivers / crypto / cavium / nitrox / nitrox_main.c

#include <linux/aer.h>
#include <linux/delay.h>
#include <linux/debugfs.h>
#include <linux/firmware.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/pci.h>
#include <linux/pci_ids.h>

#include "nitrox_dev.h"
#include "nitrox_common.h"
#include "nitrox_csr.h"

#define CNN55XX_DEV_ID	0x12
#define MAX_PF_QUEUES	64
#define UCODE_HLEN 48
#define SE_GROUP 0

#define DRIVER_VERSION "1.0"
#define FW_DIR "cavium/"
/* SE microcode */
#define SE_FW	FW_DIR "cnn55xx_se.fw"

static const char nitrox_driver_name[] = "CNN55XX";

static LIST_HEAD(ndevlist);
static DEFINE_MUTEX(devlist_lock);
static unsigned int num_devices;

/**
 * nitrox_pci_tbl - PCI Device ID Table
 */
static const struct pci_device_id nitrox_pci_tbl[] = {
	{PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
	/* required last entry */
	{0, }
};
MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);

static unsigned int qlen = DEFAULT_CMD_QLEN;
module_param(qlen, uint, 0644);
MODULE_PARM_DESC(qlen, "Command queue length - default 2048");

/**
 * struct ucode - Firmware Header
 * @id: microcode ID
 * @version: firmware version
 * @code_size: code section size
 * @raz: alignment
 * @code: code section
 */
struct ucode {
	u8 id;
	char version[VERSION_LEN - 1];
	__be32 code_size;
	u8 raz[12];
	u64 code[0];
};

/**
 * write_to_ucd_unit - Write Firmware to NITROX UCD unit
 */
static void write_to_ucd_unit(struct nitrox_device *ndev,
			      struct ucode *ucode)
{
	u32 code_size = be32_to_cpu(ucode->code_size) * 2;
	u64 offset, data;
	int i = 0;

	/*
	 * UCD structure
	 *
	 *  -------------
	 *  |    BLK 7  |
	 *  -------------
	 *  |    BLK 6  |
	 *  -------------
	 *  |    ...    |
	 *  -------------
	 *  |    BLK 0  |
	 *  -------------
	 *  Total of 8 blocks, each size 32KB
	 */

	/* set the block number */
	offset = UCD_UCODE_LOAD_BLOCK_NUM;
	nitrox_write_csr(ndev, offset, 0);

	code_size = roundup(code_size, 8);
	while (code_size) {
		data = ucode->code[i];
		/* write 8 bytes at a time */
		offset = UCD_UCODE_LOAD_IDX_DATAX(i);
		nitrox_write_csr(ndev, offset, data);
		code_size -= 8;
		i++;
	}

	/* put all SE cores in group 0 */
	offset = POM_GRP_EXECMASKX(SE_GROUP);
	nitrox_write_csr(ndev, offset, (~0ULL));

	for (i = 0; i < ndev->hw.se_cores; i++) {
		/*
		 * write block number and firware length
		 * bit:<2:0> block number
		 * bit:3 is set SE uses 32KB microcode
		 * bit:3 is clear SE uses 64KB microcode
		 */
		offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
		nitrox_write_csr(ndev, offset, 0x8);
	}
	usleep_range(300, 400);
}

static int nitrox_load_fw(struct nitrox_device *ndev, const char *fw_name)
{
	const struct firmware *fw;
	struct ucode *ucode;
	int ret;

	dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);

	ret = request_firmware(&fw, fw_name, DEV(ndev));
	if (ret < 0) {
		dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
		return ret;
	}

	ucode = (struct ucode *)fw->data;
	/* copy the firmware version */
	memcpy(ndev->hw.fw_name, ucode->version, (VERSION_LEN - 2));
	ndev->hw.fw_name[VERSION_LEN - 1] = '\0';

	write_to_ucd_unit(ndev, ucode);
	release_firmware(fw);

	set_bit(NITROX_UCODE_LOADED, &ndev->status);
	/* barrier to sync with other cpus */
	smp_mb__after_atomic();
	return 0;
}

/**
 * nitrox_add_to_devlist - add NITROX device to global device list
 * @ndev: NITROX device
 */
static int nitrox_add_to_devlist(struct nitrox_device *ndev)
{
	struct nitrox_device *dev;
	int ret = 0;

	INIT_LIST_HEAD(&ndev->list);
	refcount_set(&ndev->refcnt, 1);

	mutex_lock(&devlist_lock);
	list_for_each_entry(dev, &ndevlist, list) {
		if (dev == ndev) {
			ret = -EEXIST;
			goto unlock;
		}
	}
	ndev->idx = num_devices++;
	list_add_tail(&ndev->list, &ndevlist);
unlock:
	mutex_unlock(&devlist_lock);
	return ret;
}

/**
 * nitrox_remove_from_devlist - remove NITROX device from
 *   global device list
 * @ndev: NITROX device
 */
static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
{
	mutex_lock(&devlist_lock);
	list_del(&ndev->list);
	num_devices--;
	mutex_unlock(&devlist_lock);
}

struct nitrox_device *nitrox_get_first_device(void)
{
	struct nitrox_device *ndev = NULL;

	mutex_lock(&devlist_lock);
	list_for_each_entry(ndev, &ndevlist, list) {
		if (nitrox_ready(ndev))
			break;
	}
	mutex_unlock(&devlist_lock);
	if (!ndev)
		return NULL;

	refcount_inc(&ndev->refcnt);
	/* barrier to sync with other cpus */
	smp_mb__after_atomic();
	return ndev;
}

void nitrox_put_device(struct nitrox_device *ndev)
{
	if (!ndev)
		return;

	refcount_dec(&ndev->refcnt);
	/* barrier to sync with other cpus */
	smp_mb__after_atomic();
}

static int nitrox_reset_device(struct pci_dev *pdev)
{
	int pos = 0;

	pos = pci_save_state(pdev);
	if (pos) {
		dev_err(&pdev->dev, "Failed to save pci state\n");
		return -ENOMEM;
	}

	pos = pci_pcie_cap(pdev);
	if (!pos)
		return -ENOTTY;

	if (!pci_wait_for_pending_transaction(pdev))
		dev_err(&pdev->dev, "waiting for pending transaction\n");

	pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
	msleep(100);
	pci_restore_state(pdev);

	return 0;
}

static int nitrox_pf_sw_init(struct nitrox_device *ndev)
{
	int err;

	err = nitrox_common_sw_init(ndev);
	if (err)
		return err;

	err = nitrox_pf_init_isr(ndev);
	if (err)
		nitrox_common_sw_cleanup(ndev);

	return err;
}

static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
{
	nitrox_pf_cleanup_isr(ndev);
	nitrox_common_sw_cleanup(ndev);
}

/**
 * nitrox_bist_check - Check NITORX BIST registers status
 * @ndev: NITROX device
 */
static int nitrox_bist_check(struct nitrox_device *ndev)
{
	u64 value = 0;
	int i;

	for (i = 0; i < NR_CLUSTERS; i++) {
		value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
		value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
	}
	value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
	value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
	value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
	value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
	value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
	value += nitrox_read_csr(ndev, POM_BIST_REG);
	value += nitrox_read_csr(ndev, BMI_BIST_REG);
	value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
	value += nitrox_read_csr(ndev, BMO_BIST_REG);
	value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
	value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
	if (value)
		return -EIO;
	return 0;
}

static void nitrox_get_hwinfo(struct nitrox_device *ndev)
{
	union emu_fuse_map emu_fuse;
	u64 offset;
	int i;

	for (i = 0; i < NR_CLUSTERS; i++) {
		u8 dead_cores;

		offset = EMU_FUSE_MAPX(i);
		emu_fuse.value = nitrox_read_csr(ndev, offset);
		if (emu_fuse.s.valid) {
			dead_cores = hweight32(emu_fuse.s.ae_fuse);
			ndev->hw.ae_cores += AE_CORES_PER_CLUSTER - dead_cores;
			dead_cores = hweight16(emu_fuse.s.se_fuse);
			ndev->hw.se_cores += SE_CORES_PER_CLUSTER - dead_cores;
		}
	}
}

static int nitrox_pf_hw_init(struct nitrox_device *ndev)
{
	int err;

	err = nitrox_bist_check(ndev);
	if (err) {
		dev_err(&ndev->pdev->dev, "BIST check failed\n");
		return err;
	}
	/* get cores information */
	nitrox_get_hwinfo(ndev);

	nitrox_config_nps_unit(ndev);
	nitrox_config_pom_unit(ndev);
	nitrox_config_efl_unit(ndev);
	/* configure IO units */
	nitrox_config_bmi_unit(ndev);
	nitrox_config_bmo_unit(ndev);
	/* configure Local Buffer Cache */
	nitrox_config_lbc_unit(ndev);
	nitrox_config_rand_unit(ndev);

	/* load firmware on SE cores */
	err = nitrox_load_fw(ndev, SE_FW);
	if (err)
		return err;

	nitrox_config_emu_unit(ndev);

	return 0;
}

#if IS_ENABLED(CONFIG_DEBUG_FS)
static int registers_show(struct seq_file *s, void *v)
{
	struct nitrox_device *ndev = s->private;
	u64 offset;

	/* NPS DMA stats */
	offset = NPS_STATS_PKT_DMA_RD_CNT;
	seq_printf(s, "NPS_STATS_PKT_DMA_RD_CNT  0x%016llx\n",
		   nitrox_read_csr(ndev, offset));
	offset = NPS_STATS_PKT_DMA_WR_CNT;
	seq_printf(s, "NPS_STATS_PKT_DMA_WR_CNT  0x%016llx\n",
		   nitrox_read_csr(ndev, offset));

	/* BMI/BMO stats */
	offset = BMI_NPS_PKT_CNT;
	seq_printf(s, "BMI_NPS_PKT_CNT  0x%016llx\n",
		   nitrox_read_csr(ndev, offset));
	offset = BMO_NPS_SLC_PKT_CNT;
	seq_printf(s, "BMO_NPS_PKT_CNT  0x%016llx\n",
		   nitrox_read_csr(ndev, offset));

	return 0;
}

static int registers_open(struct inode *inode, struct file *file)
{
	return single_open(file, registers_show, inode->i_private);
}

static const struct file_operations register_fops = {
	.owner = THIS_MODULE,
	.open = registers_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = single_release,
};

static int firmware_show(struct seq_file *s, void *v)
{
	struct nitrox_device *ndev = s->private;

	seq_printf(s, "Version: %s\n", ndev->hw.fw_name);
	return 0;
}

static int firmware_open(struct inode *inode, struct file *file)
{
	return single_open(file, firmware_show, inode->i_private);
}

static const struct file_operations firmware_fops = {
	.owner = THIS_MODULE,
	.open = firmware_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = single_release,
};

static int nitrox_show(struct seq_file *s, void *v)
{
	struct nitrox_device *ndev = s->private;

	seq_printf(s, "NITROX-5 [idx: %d]\n", ndev->idx);
	seq_printf(s, "  Revision ID: 0x%0x\n", ndev->hw.revision_id);
	seq_printf(s, "  Cores [AE: %u  SE: %u]\n",
		   ndev->hw.ae_cores, ndev->hw.se_cores);
	seq_printf(s, "  Number of Queues: %u\n", ndev->nr_queues);
	seq_printf(s, "  Queue length: %u\n", ndev->qlen);
	seq_printf(s, "  Node: %u\n", ndev->node);

	return 0;
}

static int nitrox_open(struct inode *inode, struct file *file)
{
	return single_open(file, nitrox_show, inode->i_private);
}

static const struct file_operations nitrox_fops = {
	.owner = THIS_MODULE,
	.open = nitrox_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = single_release,
};

static void nitrox_debugfs_exit(struct nitrox_device *ndev)
{
	debugfs_remove_recursive(ndev->debugfs_dir);
	ndev->debugfs_dir = NULL;
}

static int nitrox_debugfs_init(struct nitrox_device *ndev)
{
	struct dentry *dir, *f;

	dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
	if (!dir)
		return -ENOMEM;

	ndev->debugfs_dir = dir;
	f = debugfs_create_file("counters", 0400, dir, ndev, &register_fops);
	if (!f)
		goto err;
	f = debugfs_create_file("firmware", 0400, dir, ndev, &firmware_fops);
	if (!f)
		goto err;
	f = debugfs_create_file("nitrox", 0400, dir, ndev, &nitrox_fops);
	if (!f)
		goto err;

	return 0;

err:
	nitrox_debugfs_exit(ndev);
	return -ENODEV;
}
#else
static int nitrox_debugfs_init(struct nitrox_device *ndev)
{
	return 0;
}

static void nitrox_debugfs_exit(struct nitrox_device *ndev)
{
}
#endif

/**
 * nitrox_probe - NITROX Initialization function.
 * @pdev: PCI device information struct
 * @id: entry in nitrox_pci_tbl
 *
 * Return: 0, if the driver is bound to the device, or
 *         a negative error if there is failure.
 */
static int nitrox_probe(struct pci_dev *pdev,
			const struct pci_device_id *id)
{
	struct nitrox_device *ndev;
	int err;

	dev_info_once(&pdev->dev, "%s driver version %s\n",
		      nitrox_driver_name, DRIVER_VERSION);

	err = pci_enable_device_mem(pdev);
	if (err)
		return err;

	/* do FLR */
	err = nitrox_reset_device(pdev);
	if (err) {
		dev_err(&pdev->dev, "FLR failed\n");
		pci_disable_device(pdev);
		return err;
	}

	if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
		dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
	} else {
		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
		if (err) {
			dev_err(&pdev->dev, "DMA configuration failed\n");
			pci_disable_device(pdev);
			return err;
		}
	}

	err = pci_request_mem_regions(pdev, nitrox_driver_name);
	if (err) {
		pci_disable_device(pdev);
		return err;
	}
	pci_set_master(pdev);

	ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
	if (!ndev)
		goto ndev_fail;

	pci_set_drvdata(pdev, ndev);
	ndev->pdev = pdev;

	/* add to device list */
	nitrox_add_to_devlist(ndev);

	ndev->hw.vendor_id = pdev->vendor;
	ndev->hw.device_id = pdev->device;
	ndev->hw.revision_id = pdev->revision;
	/* command timeout in jiffies */
	ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
	ndev->node = dev_to_node(&pdev->dev);
	if (ndev->node == NUMA_NO_NODE)
		ndev->node = 0;

	ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
				 pci_resource_len(pdev, 0));
	if (!ndev->bar_addr) {
		err = -EIO;
		goto ioremap_err;
	}
	/* allocate command queus based on cpus, max queues are 64 */
	ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
	ndev->qlen = qlen;

	err = nitrox_pf_sw_init(ndev);
	if (err)
		goto ioremap_err;

	err = nitrox_pf_hw_init(ndev);
	if (err)
		goto pf_hw_fail;

	err = nitrox_debugfs_init(ndev);
	if (err)
		goto pf_hw_fail;

	set_bit(NITROX_READY, &ndev->status);
	/* barrier to sync with other cpus */
	smp_mb__after_atomic();

	err = nitrox_crypto_register();
	if (err)
		goto crypto_fail;

	return 0;

crypto_fail:
	nitrox_debugfs_exit(ndev);
	clear_bit(NITROX_READY, &ndev->status);
	/* barrier to sync with other cpus */
	smp_mb__after_atomic();
pf_hw_fail:
	nitrox_pf_sw_cleanup(ndev);
ioremap_err:
	nitrox_remove_from_devlist(ndev);
	kfree(ndev);
	pci_set_drvdata(pdev, NULL);
ndev_fail:
	pci_release_mem_regions(pdev);
	pci_disable_device(pdev);
	return err;
}

/**
 * nitrox_remove - Unbind the driver from the device.
 * @pdev: PCI device information struct
 */
static void nitrox_remove(struct pci_dev *pdev)
{
	struct nitrox_device *ndev = pci_get_drvdata(pdev);

	if (!ndev)
		return;

	if (!refcount_dec_and_test(&ndev->refcnt)) {
		dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
			refcount_read(&ndev->refcnt));
		return;
	}

	dev_info(DEV(ndev), "Removing Device %x:%x\n",
		 ndev->hw.vendor_id, ndev->hw.device_id);

	clear_bit(NITROX_READY, &ndev->status);
	/* barrier to sync with other cpus */
	smp_mb__after_atomic();

	nitrox_remove_from_devlist(ndev);
	nitrox_crypto_unregister();
	nitrox_debugfs_exit(ndev);
	nitrox_pf_sw_cleanup(ndev);

	iounmap(ndev->bar_addr);
	kfree(ndev);

	pci_set_drvdata(pdev, NULL);
	pci_release_mem_regions(pdev);
	pci_disable_device(pdev);
}

static void nitrox_shutdown(struct pci_dev *pdev)
{
	pci_set_drvdata(pdev, NULL);
	pci_release_mem_regions(pdev);
	pci_disable_device(pdev);
}

static struct pci_driver nitrox_driver = {
	.name = nitrox_driver_name,
	.id_table = nitrox_pci_tbl,
	.probe = nitrox_probe,
	.remove	= nitrox_remove,
	.shutdown = nitrox_shutdown,
};

module_pci_driver(nitrox_driver);

MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
MODULE_LICENSE("GPL");
MODULE_VERSION(DRIVER_VERSION);
MODULE_FIRMWARE(SE_FW);
Commit	Line	Data
14fa93cd SJ	1	#include <linux/aer.h>
14fa93cd SJ	2	#include <linux/delay.h>
086eac9e	3	#include <linux/debugfs.h>
14fa93cd SJ	4	#include <linux/firmware.h>
	5	#include <linux/list.h>
	6	#include <linux/module.h>
	7	#include <linux/mutex.h>
	8	#include <linux/pci.h>
	9	#include <linux/pci_ids.h>
	10
	11	#include "nitrox_dev.h"
	12	#include "nitrox_common.h"
	13	#include "nitrox_csr.h"
	14
	15	#define CNN55XX_DEV_ID 0x12
	16	#define MAX_PF_QUEUES 64
	17	#define UCODE_HLEN 48
	18	#define SE_GROUP 0
	19
	20	#define DRIVER_VERSION "1.0"
440bf347	21	#define FW_DIR "cavium/"
14fa93cd	22	/* SE microcode */
440bf347	23	#define SE_FW FW_DIR "cnn55xx_se.fw"
14fa93cd SJ	24
	25	static const char nitrox_driver_name[] = "CNN55XX";
	26
	27	static LIST_HEAD(ndevlist);
	28	static DEFINE_MUTEX(devlist_lock);
	29	static unsigned int num_devices;
	30
	31	/**
	32	* nitrox_pci_tbl - PCI Device ID Table
	33	*/
	34	static const struct pci_device_id nitrox_pci_tbl[] = {
	35	{PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
	36	/* required last entry */
	37	{0, }
	38	};
	39	MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);
	40
	41	static unsigned int qlen = DEFAULT_CMD_QLEN;
	42	module_param(qlen, uint, 0644);
	43	MODULE_PARM_DESC(qlen, "Command queue length - default 2048");
	44
	45	/**
	46	* struct ucode - Firmware Header
	47	* @id: microcode ID
	48	* @version: firmware version
	49	* @code_size: code section size
	50	* @raz: alignment
	51	* @code: code section
	52	*/
	53	struct ucode {
	54	u8 id;
	55	char version[VERSION_LEN - 1];
	56	__be32 code_size;
	57	u8 raz[12];
	58	u64 code[0];
	59	};
	60
	61	/**
	62	* write_to_ucd_unit - Write Firmware to NITROX UCD unit
	63	*/
	64	static void write_to_ucd_unit(struct nitrox_device *ndev,
	65	struct ucode *ucode)
	66	{
	67	u32 code_size = be32_to_cpu(ucode->code_size) * 2;
	68	u64 offset, data;
	69	int i = 0;
	70
	71	/*
	72	* UCD structure
	73	*
	74	* -------------
	75	* \| BLK 7 \|
	76	* -------------
	77	* \| BLK 6 \|
	78	* -------------
	79	* \| ... \|
	80	* -------------
	81	* \| BLK 0 \|
	82	* -------------
	83	* Total of 8 blocks, each size 32KB
	84	*/
	85
	86	/* set the block number */
	87	offset = UCD_UCODE_LOAD_BLOCK_NUM;
88	nitrox_write_csr(ndev, offset, 0);
89
90	code_size = roundup(code_size, 8);
91	while (code_size) {
92	data = ucode->code[i];
93	/* write 8 bytes at a time */
94	offset = UCD_UCODE_LOAD_IDX_DATAX(i);
95	nitrox_write_csr(ndev, offset, data);
96	code_size -= 8;
97	i++;
98	}
99
100	/* put all SE cores in group 0 */
101	offset = POM_GRP_EXECMASKX(SE_GROUP);
102	nitrox_write_csr(ndev, offset, (~0ULL));
103
104	for (i = 0; i < ndev->hw.se_cores; i++) {
105	/*
106	* write block number and firware length
107	* bit:<2:0> block number
108	* bit:3 is set SE uses 32KB microcode
109	* bit:3 is clear SE uses 64KB microcode
110	*/
111	offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
112	nitrox_write_csr(ndev, offset, 0x8);
113	}
114	usleep_range(300, 400);
115	}
116
117	static int nitrox_load_fw(struct nitrox_device ndev, const char fw_name)
118	{
119	const struct firmware *fw;
120	struct ucode *ucode;
121	int ret;
122
123	dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
124
125	ret = request_firmware(&fw, fw_name, DEV(ndev));
126	if (ret < 0) {
127	dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
128	return ret;
129	}
130
131	ucode = (struct ucode *)fw->data;
132	/* copy the firmware version */
133	memcpy(ndev->hw.fw_name, ucode->version, (VERSION_LEN - 2));
134	ndev->hw.fw_name[VERSION_LEN - 1] = '\0';
135
136	write_to_ucd_unit(ndev, ucode);
137	release_firmware(fw);
138
139	set_bit(NITROX_UCODE_LOADED, &ndev->status);
140	/* barrier to sync with other cpus */
141	smp_mb__after_atomic();
142	return 0;
143	}
144
145	/**
146	* nitrox_add_to_devlist - add NITROX device to global device list
147	* @ndev: NITROX device
148	*/
149	static int nitrox_add_to_devlist(struct nitrox_device *ndev)
150	{
151	struct nitrox_device *dev;
152	int ret = 0;
153
154	INIT_LIST_HEAD(&ndev->list);
155	refcount_set(&ndev->refcnt, 1);
156
157	mutex_lock(&devlist_lock);
158	list_for_each_entry(dev, &ndevlist, list) {
159	if (dev == ndev) {
160	ret = -EEXIST;
161	goto unlock;
162	}
163	}
164	ndev->idx = num_devices++;
165	list_add_tail(&ndev->list, &ndevlist);
166	unlock:
167	mutex_unlock(&devlist_lock);
168	return ret;
169	}
170
171	/**
172	* nitrox_remove_from_devlist - remove NITROX device from
173	* global device list
174	* @ndev: NITROX device
175	*/
176	static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
177	{
178	mutex_lock(&devlist_lock);
179	list_del(&ndev->list);
180	num_devices--;
181	mutex_unlock(&devlist_lock);
182	}
183
f2663872 SJ	184	struct nitrox_device *nitrox_get_first_device(void)
	185	{
	186	struct nitrox_device *ndev = NULL;
	187
	188	mutex_lock(&devlist_lock);
	189	list_for_each_entry(ndev, &ndevlist, list) {
	190	if (nitrox_ready(ndev))
	191	break;
	192	}
	193	mutex_unlock(&devlist_lock);
	194	if (!ndev)
	195	return NULL;
	196
	197	refcount_inc(&ndev->refcnt);
	198	/* barrier to sync with other cpus */
	199	smp_mb__after_atomic();
	200	return ndev;
	201	}
	202
	203	void nitrox_put_device(struct nitrox_device *ndev)
	204	{
	205	if (!ndev)
	206	return;
	207
	208	refcount_dec(&ndev->refcnt);
	209	/* barrier to sync with other cpus */
	210	smp_mb__after_atomic();
	211	}
	212
14fa93cd SJ	213	static int nitrox_reset_device(struct pci_dev *pdev)
	214	{
	215	int pos = 0;
	216
	217	pos = pci_save_state(pdev);
	218	if (pos) {
	219	dev_err(&pdev->dev, "Failed to save pci state\n");
	220	return -ENOMEM;
	221	}
	222
	223	pos = pci_pcie_cap(pdev);
	224	if (!pos)
	225	return -ENOTTY;
	226
	227	if (!pci_wait_for_pending_transaction(pdev))
	228	dev_err(&pdev->dev, "waiting for pending transaction\n");
	229
	230	pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
	231	msleep(100);
	232	pci_restore_state(pdev);
	233
	234	return 0;
	235	}
	236
	237	static int nitrox_pf_sw_init(struct nitrox_device *ndev)
	238	{
	239	int err;
	240
	241	err = nitrox_common_sw_init(ndev);
	242	if (err)
	243	return err;
	244
	245	err = nitrox_pf_init_isr(ndev);
	246	if (err)
	247	nitrox_common_sw_cleanup(ndev);
	248
	249	return err;
	250	}
	251
	252	static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
	253	{
	254	nitrox_pf_cleanup_isr(ndev);
	255	nitrox_common_sw_cleanup(ndev);
	256	}
	257
	258	/**
	259	* nitrox_bist_check - Check NITORX BIST registers status
	260	* @ndev: NITROX device
	261	*/
	262	static int nitrox_bist_check(struct nitrox_device *ndev)
	263	{
	264	u64 value = 0;
	265	int i;
	266
	267	for (i = 0; i < NR_CLUSTERS; i++) {
	268	value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
	269	value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
	270	}
	271	value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
	272	value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
	273	value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
	274	value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
	275	value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
	276	value += nitrox_read_csr(ndev, POM_BIST_REG);
277	value += nitrox_read_csr(ndev, BMI_BIST_REG);
278	value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
279	value += nitrox_read_csr(ndev, BMO_BIST_REG);
280	value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
281	value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
282	if (value)
283	return -EIO;
284	return 0;
285	}
286
287	static void nitrox_get_hwinfo(struct nitrox_device *ndev)
288	{
289	union emu_fuse_map emu_fuse;
290	u64 offset;
291	int i;
292
293	for (i = 0; i < NR_CLUSTERS; i++) {
294	u8 dead_cores;
295
296	offset = EMU_FUSE_MAPX(i);
297	emu_fuse.value = nitrox_read_csr(ndev, offset);
298	if (emu_fuse.s.valid) {
299	dead_cores = hweight32(emu_fuse.s.ae_fuse);
300	ndev->hw.ae_cores += AE_CORES_PER_CLUSTER - dead_cores;
301	dead_cores = hweight16(emu_fuse.s.se_fuse);
302	ndev->hw.se_cores += SE_CORES_PER_CLUSTER - dead_cores;
303	}
304	}
305	}
306
307	static int nitrox_pf_hw_init(struct nitrox_device *ndev)
308	{
309	int err;
310
311	err = nitrox_bist_check(ndev);
312	if (err) {
313	dev_err(&ndev->pdev->dev, "BIST check failed\n");
314	return err;
315	}
316	/* get cores information */
317	nitrox_get_hwinfo(ndev);
318
319	nitrox_config_nps_unit(ndev);
320	nitrox_config_pom_unit(ndev);
321	nitrox_config_efl_unit(ndev);
322	/* configure IO units */
323	nitrox_config_bmi_unit(ndev);
324	nitrox_config_bmo_unit(ndev);
325	/* configure Local Buffer Cache */
326	nitrox_config_lbc_unit(ndev);
327	nitrox_config_rand_unit(ndev);
328
329	/* load firmware on SE cores */
330	err = nitrox_load_fw(ndev, SE_FW);
331	if (err)
332	return err;
333
334	nitrox_config_emu_unit(ndev);
335
336	return 0;
337	}
338
086eac9e SJ	339	#if IS_ENABLED(CONFIG_DEBUG_FS)
	340	static int registers_show(struct seq_file s, void v)
	341	{
	342	struct nitrox_device *ndev = s->private;
	343	u64 offset;
	344
	345	/* NPS DMA stats */
	346	offset = NPS_STATS_PKT_DMA_RD_CNT;
	347	seq_printf(s, "NPS_STATS_PKT_DMA_RD_CNT 0x%016llx\n",
	348	nitrox_read_csr(ndev, offset));
	349	offset = NPS_STATS_PKT_DMA_WR_CNT;
	350	seq_printf(s, "NPS_STATS_PKT_DMA_WR_CNT 0x%016llx\n",
	351	nitrox_read_csr(ndev, offset));
	352
	353	/* BMI/BMO stats */
	354	offset = BMI_NPS_PKT_CNT;
	355	seq_printf(s, "BMI_NPS_PKT_CNT 0x%016llx\n",
	356	nitrox_read_csr(ndev, offset));
	357	offset = BMO_NPS_SLC_PKT_CNT;
	358	seq_printf(s, "BMO_NPS_PKT_CNT 0x%016llx\n",
	359	nitrox_read_csr(ndev, offset));
	360
	361	return 0;
	362	}
	363
	364	static int registers_open(struct inode inode, struct file file)
	365	{
	366	return single_open(file, registers_show, inode->i_private);
	367	}
	368
	369	static const struct file_operations register_fops = {
	370	.owner = THIS_MODULE,
	371	.open = registers_open,
	372	.read = seq_read,
	373	.llseek = seq_lseek,
	374	.release = single_release,
	375	};
	376
	377	static int firmware_show(struct seq_file s, void v)
	378	{
	379	struct nitrox_device *ndev = s->private;
	380
	381	seq_printf(s, "Version: %s\n", ndev->hw.fw_name);
	382	return 0;
	383	}
	384
	385	static int firmware_open(struct inode inode, struct file file)
	386	{
	387	return single_open(file, firmware_show, inode->i_private);
	388	}
	389
	390	static const struct file_operations firmware_fops = {
	391	.owner = THIS_MODULE,
	392	.open = firmware_open,
	393	.read = seq_read,
	394	.llseek = seq_lseek,
	395	.release = single_release,
	396	};
	397
	398	static int nitrox_show(struct seq_file s, void v)
	399	{
	400	struct nitrox_device *ndev = s->private;
	401
	402	seq_printf(s, "NITROX-5 [idx: %d]\n", ndev->idx);
7c742df5	403	seq_printf(s, " Revision ID: 0x%0x\n", ndev->hw.revision_id);
086eac9e SJ	404	seq_printf(s, " Cores [AE: %u SE: %u]\n",
	405	ndev->hw.ae_cores, ndev->hw.se_cores);
	406	seq_printf(s, " Number of Queues: %u\n", ndev->nr_queues);
	407	seq_printf(s, " Queue length: %u\n", ndev->qlen);
	408	seq_printf(s, " Node: %u\n", ndev->node);
	409
	410	return 0;
	411	}
	412
	413	static int nitrox_open(struct inode inode, struct file file)
	414	{
	415	return single_open(file, nitrox_show, inode->i_private);
	416	}
	417
	418	static const struct file_operations nitrox_fops = {
	419	.owner = THIS_MODULE,
	420	.open = nitrox_open,
	421	.read = seq_read,
	422	.llseek = seq_lseek,
	423	.release = single_release,
	424	};
	425
	426	static void nitrox_debugfs_exit(struct nitrox_device *ndev)
	427	{
	428	debugfs_remove_recursive(ndev->debugfs_dir);
	429	ndev->debugfs_dir = NULL;
	430	}
	431
	432	static int nitrox_debugfs_init(struct nitrox_device *ndev)
	433	{
	434	struct dentry dir, f;
	435
	436	dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
	437	if (!dir)
	438	return -ENOMEM;
	439
	440	ndev->debugfs_dir = dir;
	441	f = debugfs_create_file("counters", 0400, dir, ndev, &register_fops);
	442	if (!f)
	443	goto err;
	444	f = debugfs_create_file("firmware", 0400, dir, ndev, &firmware_fops);
	445	if (!f)
	446	goto err;
	447	f = debugfs_create_file("nitrox", 0400, dir, ndev, &nitrox_fops);
	448	if (!f)
	449	goto err;
	450
	451	return 0;
	452
	453	err:
	454	nitrox_debugfs_exit(ndev);
	455	return -ENODEV;
	456	}
	457	#else
	458	static int nitrox_debugfs_init(struct nitrox_device *ndev)
	459	{
	460	return 0;
	461	}
	462
	463	static void nitrox_debugfs_exit(struct nitrox_device *ndev)
	464	{
	465	}
	466	#endif
	467
14fa93cd SJ	468	/**
	469	* nitrox_probe - NITROX Initialization function.
	470	* @pdev: PCI device information struct
	471	* @id: entry in nitrox_pci_tbl
	472	*
	473	* Return: 0, if the driver is bound to the device, or
	474	* a negative error if there is failure.
	475	*/
	476	static int nitrox_probe(struct pci_dev *pdev,
	477	const struct pci_device_id *id)
	478	{
	479	struct nitrox_device *ndev;
	480	int err;
	481
	482	dev_info_once(&pdev->dev, "%s driver version %s\n",
	483	nitrox_driver_name, DRIVER_VERSION);
	484
	485	err = pci_enable_device_mem(pdev);
	486	if (err)
	487	return err;
	488
	489	/* do FLR */
	490	err = nitrox_reset_device(pdev);
	491	if (err) {
	492	dev_err(&pdev->dev, "FLR failed\n");
	493	pci_disable_device(pdev);
	494	return err;
	495	}
	496
	497	if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
	498	dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
	499	} else {
	500	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
	501	if (err) {
	502	dev_err(&pdev->dev, "DMA configuration failed\n");
	503	pci_disable_device(pdev);
	504	return err;
	505	}
	506	}
	507
	508	err = pci_request_mem_regions(pdev, nitrox_driver_name);
	509	if (err) {
	510	pci_disable_device(pdev);
	511	return err;
	512	}
	513	pci_set_master(pdev);
	514
	515	ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
	516	if (!ndev)
	517	goto ndev_fail;
	518
	519	pci_set_drvdata(pdev, ndev);
	520	ndev->pdev = pdev;
	521
	522	/* add to device list */
	523	nitrox_add_to_devlist(ndev);
	524
	525	ndev->hw.vendor_id = pdev->vendor;
	526	ndev->hw.device_id = pdev->device;
	527	ndev->hw.revision_id = pdev->revision;
	528	/* command timeout in jiffies */
	529	ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
	530	ndev->node = dev_to_node(&pdev->dev);
	531	if (ndev->node == NUMA_NO_NODE)
532	ndev->node = 0;
533
534	ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
535	pci_resource_len(pdev, 0));
536	if (!ndev->bar_addr) {
537	err = -EIO;
538	goto ioremap_err;
539	}
540	/* allocate command queus based on cpus, max queues are 64 */
541	ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
542	ndev->qlen = qlen;
543
544	err = nitrox_pf_sw_init(ndev);
545	if (err)
546	goto ioremap_err;
547
548	err = nitrox_pf_hw_init(ndev);
549	if (err)
550	goto pf_hw_fail;
551
086eac9e SJ	552	err = nitrox_debugfs_init(ndev);
	553	if (err)
	554	goto pf_hw_fail;
	555
14fa93cd SJ	556	set_bit(NITROX_READY, &ndev->status);
	557	/* barrier to sync with other cpus */
	558	smp_mb__after_atomic();
f2663872 SJ	559
	560	err = nitrox_crypto_register();
	561	if (err)
	562	goto crypto_fail;
	563
14fa93cd SJ	564	return 0;
14fa93cd SJ	565
f2663872 SJ	566	crypto_fail:
	567	nitrox_debugfs_exit(ndev);
	568	clear_bit(NITROX_READY, &ndev->status);
	569	/* barrier to sync with other cpus */
	570	smp_mb__after_atomic();
14fa93cd SJ	571	pf_hw_fail:
	572	nitrox_pf_sw_cleanup(ndev);
	573	ioremap_err:
	574	nitrox_remove_from_devlist(ndev);
	575	kfree(ndev);
	576	pci_set_drvdata(pdev, NULL);
	577	ndev_fail:
	578	pci_release_mem_regions(pdev);
	579	pci_disable_device(pdev);
	580	return err;
	581	}
	582
	583	/**
	584	* nitrox_remove - Unbind the driver from the device.
	585	* @pdev: PCI device information struct
	586	*/
	587	static void nitrox_remove(struct pci_dev *pdev)
	588	{
	589	struct nitrox_device *ndev = pci_get_drvdata(pdev);
	590
	591	if (!ndev)
	592	return;
	593
	594	if (!refcount_dec_and_test(&ndev->refcnt)) {
	595	dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
	596	refcount_read(&ndev->refcnt));
	597	return;
	598	}
	599
	600	dev_info(DEV(ndev), "Removing Device %x:%x\n",
	601	ndev->hw.vendor_id, ndev->hw.device_id);
	602
	603	clear_bit(NITROX_READY, &ndev->status);
	604	/* barrier to sync with other cpus */
	605	smp_mb__after_atomic();
	606
	607	nitrox_remove_from_devlist(ndev);
f2663872	608	nitrox_crypto_unregister();
086eac9e	609	nitrox_debugfs_exit(ndev);
14fa93cd SJ	610	nitrox_pf_sw_cleanup(ndev);
	611
	612	iounmap(ndev->bar_addr);
	613	kfree(ndev);
	614
	615	pci_set_drvdata(pdev, NULL);
	616	pci_release_mem_regions(pdev);
	617	pci_disable_device(pdev);
	618	}
	619
	620	static void nitrox_shutdown(struct pci_dev *pdev)
	621	{
	622	pci_set_drvdata(pdev, NULL);
	623	pci_release_mem_regions(pdev);
	624	pci_disable_device(pdev);
	625	}
	626
	627	static struct pci_driver nitrox_driver = {
	628	.name = nitrox_driver_name,
	629	.id_table = nitrox_pci_tbl,
	630	.probe = nitrox_probe,
	631	.remove = nitrox_remove,
	632	.shutdown = nitrox_shutdown,
	633	};
	634
	635	module_pci_driver(nitrox_driver);
	636
	637	MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
	638	MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
	639	MODULE_LICENSE("GPL");
	640	MODULE_VERSION(DRIVER_VERSION);
	641	MODULE_FIRMWARE(SE_FW);