[mirror_ubuntu-zesty-kernel.git] / drivers / nvme / host / lightnvm.c

/*
 * nvme-lightnvm.c - LightNVM NVMe device
 *
 * Copyright (C) 2014-2015 IT University of Copenhagen
 * Initial release: Matias Bjorling <mb@lightnvm.io>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License version
 * 2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; see the file COPYING.  If not, write to
 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139,
 * USA.
 *
 */

#include "nvme.h"

#ifdef CONFIG_NVM

#include <linux/nvme.h>
#include <linux/bitops.h>
#include <linux/lightnvm.h>
#include <linux/vmalloc.h>

enum nvme_nvm_admin_opcode {
	nvme_nvm_admin_identity		= 0xe2,
	nvme_nvm_admin_get_l2p_tbl	= 0xea,
	nvme_nvm_admin_get_bb_tbl	= 0xf2,
	nvme_nvm_admin_set_bb_tbl	= 0xf1,
};

struct nvme_nvm_hb_rw {
	__u8			opcode;
	__u8			flags;
	__u16			command_id;
	__le32			nsid;
	__u64			rsvd2;
	__le64			metadata;
	__le64			prp1;
	__le64			prp2;
	__le64			spba;
	__le16			length;
	__le16			control;
	__le32			dsmgmt;
	__le64			slba;
};

struct nvme_nvm_ph_rw {
	__u8			opcode;
	__u8			flags;
	__u16			command_id;
	__le32			nsid;
	__u64			rsvd2;
	__le64			metadata;
	__le64			prp1;
	__le64			prp2;
	__le64			spba;
	__le16			length;
	__le16			control;
	__le32			dsmgmt;
	__le64			resv;
};

struct nvme_nvm_identity {
	__u8			opcode;
	__u8			flags;
	__u16			command_id;
	__le32			nsid;
	__u64			rsvd[2];
	__le64			prp1;
	__le64			prp2;
	__le32			chnl_off;
	__u32			rsvd11[5];
};

struct nvme_nvm_l2ptbl {
	__u8			opcode;
	__u8			flags;
	__u16			command_id;
	__le32			nsid;
	__le32			cdw2[4];
	__le64			prp1;
	__le64			prp2;
	__le64			slba;
	__le32			nlb;
	__le16			cdw14[6];
};

struct nvme_nvm_getbbtbl {
	__u8			opcode;
	__u8			flags;
	__u16			command_id;
	__le32			nsid;
	__u64			rsvd[2];
	__le64			prp1;
	__le64			prp2;
	__le64			spba;
	__u32			rsvd4[4];
};

struct nvme_nvm_setbbtbl {
	__u8			opcode;
	__u8			flags;
	__u16			command_id;
	__le32			nsid;
	__le64			rsvd[2];
	__le64			prp1;
	__le64			prp2;
	__le64			spba;
	__le16			nlb;
	__u8			value;
	__u8			rsvd3;
	__u32			rsvd4[3];
};

struct nvme_nvm_erase_blk {
	__u8			opcode;
	__u8			flags;
	__u16			command_id;
	__le32			nsid;
	__u64			rsvd[2];
	__le64			prp1;
	__le64			prp2;
	__le64			spba;
	__le16			length;
	__le16			control;
	__le32			dsmgmt;
	__le64			resv;
};

struct nvme_nvm_command {
	union {
		struct nvme_common_command common;
		struct nvme_nvm_identity identity;
		struct nvme_nvm_hb_rw hb_rw;
		struct nvme_nvm_ph_rw ph_rw;
		struct nvme_nvm_l2ptbl l2p;
		struct nvme_nvm_getbbtbl get_bb;
		struct nvme_nvm_setbbtbl set_bb;
		struct nvme_nvm_erase_blk erase;
	};
};

struct nvme_nvm_id_group {
	__u8			mtype;
	__u8			fmtype;
	__le16			res16;
	__u8			num_ch;
	__u8			num_lun;
	__u8			num_pln;
	__u8			rsvd1;
	__le16			num_blk;
	__le16			num_pg;
	__le16			fpg_sz;
	__le16			csecs;
	__le16			sos;
	__le16			rsvd2;
	__le32			trdt;
	__le32			trdm;
	__le32			tprt;
	__le32			tprm;
	__le32			tbet;
	__le32			tbem;
	__le32			mpos;
	__le32			mccap;
	__le16			cpar;
	__u8			reserved[906];
} __packed;

struct nvme_nvm_addr_format {
	__u8			ch_offset;
	__u8			ch_len;
	__u8			lun_offset;
	__u8			lun_len;
	__u8			pln_offset;
	__u8			pln_len;
	__u8			blk_offset;
	__u8			blk_len;
	__u8			pg_offset;
	__u8			pg_len;
	__u8			sect_offset;
	__u8			sect_len;
	__u8			res[4];
} __packed;

struct nvme_nvm_id {
	__u8			ver_id;
	__u8			vmnt;
	__u8			cgrps;
	__u8			res;
	__le32			cap;
	__le32			dom;
	struct nvme_nvm_addr_format ppaf;
	__u8			resv[228];
	struct nvme_nvm_id_group groups[4];
} __packed;

struct nvme_nvm_bb_tbl {
	__u8	tblid[4];
	__le16	verid;
	__le16	revid;
	__le32	rvsd1;
	__le32	tblks;
	__le32	tfact;
	__le32	tgrown;
	__le32	tdresv;
	__le32	thresv;
	__le32	rsvd2[8];
	__u8	blk[0];
};

/*
 * Check we didn't inadvertently grow the command struct
 */
static inline void _nvme_nvm_check_size(void)
{
	BUILD_BUG_ON(sizeof(struct nvme_nvm_identity) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_nvm_hb_rw) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_nvm_ph_rw) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_nvm_getbbtbl) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_nvm_setbbtbl) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_nvm_l2ptbl) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_nvm_erase_blk) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_nvm_id_group) != 960);
	BUILD_BUG_ON(sizeof(struct nvme_nvm_addr_format) != 128);
	BUILD_BUG_ON(sizeof(struct nvme_nvm_id) != 4096);
	BUILD_BUG_ON(sizeof(struct nvme_nvm_bb_tbl) != 512);
}

static int init_grps(struct nvm_id *nvm_id, struct nvme_nvm_id *nvme_nvm_id)
{
	struct nvme_nvm_id_group *src;
	struct nvm_id_group *dst;
	int i, end;

	end = min_t(u32, 4, nvm_id->cgrps);

	for (i = 0; i < end; i++) {
		src = &nvme_nvm_id->groups[i];
		dst = &nvm_id->groups[i];

		dst->mtype = src->mtype;
		dst->fmtype = src->fmtype;
		dst->num_ch = src->num_ch;
		dst->num_lun = src->num_lun;
		dst->num_pln = src->num_pln;

		dst->num_pg = le16_to_cpu(src->num_pg);
		dst->num_blk = le16_to_cpu(src->num_blk);
		dst->fpg_sz = le16_to_cpu(src->fpg_sz);
		dst->csecs = le16_to_cpu(src->csecs);
		dst->sos = le16_to_cpu(src->sos);

		dst->trdt = le32_to_cpu(src->trdt);
		dst->trdm = le32_to_cpu(src->trdm);
		dst->tprt = le32_to_cpu(src->tprt);
		dst->tprm = le32_to_cpu(src->tprm);
		dst->tbet = le32_to_cpu(src->tbet);
		dst->tbem = le32_to_cpu(src->tbem);
		dst->mpos = le32_to_cpu(src->mpos);
		dst->mccap = le32_to_cpu(src->mccap);

		dst->cpar = le16_to_cpu(src->cpar);
	}

	return 0;
}

static int nvme_nvm_identity(struct request_queue *q, struct nvm_id *nvm_id)
{
	struct nvme_ns *ns = q->queuedata;
	struct nvme_dev *dev = ns->dev;
	struct nvme_nvm_id *nvme_nvm_id;
	struct nvme_nvm_command c = {};
	int ret;

	c.identity.opcode = nvme_nvm_admin_identity;
	c.identity.nsid = cpu_to_le32(ns->ns_id);
	c.identity.chnl_off = 0;

	nvme_nvm_id = kmalloc(sizeof(struct nvme_nvm_id), GFP_KERNEL);
	if (!nvme_nvm_id)
		return -ENOMEM;

	ret = nvme_submit_sync_cmd(dev->admin_q, (struct nvme_command *)&c,
				nvme_nvm_id, sizeof(struct nvme_nvm_id));
	if (ret) {
		ret = -EIO;
		goto out;
	}

	nvm_id->ver_id = nvme_nvm_id->ver_id;
	nvm_id->vmnt = nvme_nvm_id->vmnt;
	nvm_id->cgrps = nvme_nvm_id->cgrps;
	nvm_id->cap = le32_to_cpu(nvme_nvm_id->cap);
	nvm_id->dom = le32_to_cpu(nvme_nvm_id->dom);
	memcpy(&nvm_id->ppaf, &nvme_nvm_id->ppaf,
					sizeof(struct nvme_nvm_addr_format));

	ret = init_grps(nvm_id, nvme_nvm_id);
out:
	kfree(nvme_nvm_id);
	return ret;
}

static int nvme_nvm_get_l2p_tbl(struct request_queue *q, u64 slba, u32 nlb,
				nvm_l2p_update_fn *update_l2p, void *priv)
{
	struct nvme_ns *ns = q->queuedata;
	struct nvme_dev *dev = ns->dev;
	struct nvme_nvm_command c = {};
	u32 len = queue_max_hw_sectors(dev->admin_q) << 9;
	u32 nlb_pr_rq = len / sizeof(u64);
	u64 cmd_slba = slba;
	void *entries;
	int ret = 0;

	c.l2p.opcode = nvme_nvm_admin_get_l2p_tbl;
	c.l2p.nsid = cpu_to_le32(ns->ns_id);
	entries = kmalloc(len, GFP_KERNEL);
	if (!entries)
		return -ENOMEM;

	while (nlb) {
		u32 cmd_nlb = min(nlb_pr_rq, nlb);

		c.l2p.slba = cpu_to_le64(cmd_slba);
		c.l2p.nlb = cpu_to_le32(cmd_nlb);

		ret = nvme_submit_sync_cmd(dev->admin_q,
				(struct nvme_command *)&c, entries, len);
		if (ret) {
			dev_err(dev->dev, "L2P table transfer failed (%d)\n",
									ret);
			ret = -EIO;
			goto out;
		}

		if (update_l2p(cmd_slba, cmd_nlb, entries, priv)) {
			ret = -EINTR;
			goto out;
		}

		cmd_slba += cmd_nlb;
		nlb -= cmd_nlb;
	}

out:
	kfree(entries);
	return ret;
}

static int nvme_nvm_get_bb_tbl(struct request_queue *q, struct ppa_addr ppa,
				int nr_blocks, nvm_bb_update_fn *update_bbtbl,
				void *priv)
{
	struct nvme_ns *ns = q->queuedata;
	struct nvme_dev *dev = ns->dev;
	struct nvme_nvm_command c = {};
	struct nvme_nvm_bb_tbl *bb_tbl;
	int tblsz = sizeof(struct nvme_nvm_bb_tbl) + nr_blocks;
	int ret = 0;

	c.get_bb.opcode = nvme_nvm_admin_get_bb_tbl;
	c.get_bb.nsid = cpu_to_le32(ns->ns_id);
	c.get_bb.spba = cpu_to_le64(ppa.ppa);

	bb_tbl = kzalloc(tblsz, GFP_KERNEL);
	if (!bb_tbl)
		return -ENOMEM;

	ret = nvme_submit_sync_cmd(dev->admin_q, (struct nvme_command *)&c,
								bb_tbl, tblsz);
	if (ret) {
		dev_err(dev->dev, "get bad block table failed (%d)\n", ret);
		ret = -EIO;
		goto out;
	}

	if (bb_tbl->tblid[0] != 'B' || bb_tbl->tblid[1] != 'B' ||
		bb_tbl->tblid[2] != 'L' || bb_tbl->tblid[3] != 'T') {
		dev_err(dev->dev, "bbt format mismatch\n");
		ret = -EINVAL;
		goto out;
	}

	if (le16_to_cpu(bb_tbl->verid) != 1) {
		ret = -EINVAL;
		dev_err(dev->dev, "bbt version not supported\n");
		goto out;
	}

	if (le32_to_cpu(bb_tbl->tblks) != nr_blocks) {
		ret = -EINVAL;
		dev_err(dev->dev, "bbt unsuspected blocks returned (%u!=%u)",
					le32_to_cpu(bb_tbl->tblks), nr_blocks);
		goto out;
	}

	ret = update_bbtbl(ppa, nr_blocks, bb_tbl->blk, priv);
	if (ret) {
		ret = -EINTR;
		goto out;
	}

out:
	kfree(bb_tbl);
	return ret;
}

static int nvme_nvm_set_bb_tbl(struct request_queue *q, struct nvm_rq *rqd,
								int type)
{
	struct nvme_ns *ns = q->queuedata;
	struct nvme_dev *dev = ns->dev;
	struct nvme_nvm_command c = {};
	int ret = 0;

	c.set_bb.opcode = nvme_nvm_admin_set_bb_tbl;
	c.set_bb.nsid = cpu_to_le32(ns->ns_id);
	c.set_bb.spba = cpu_to_le64(rqd->ppa_addr.ppa);
	c.set_bb.nlb = cpu_to_le16(rqd->nr_pages - 1);
	c.set_bb.value = type;

	ret = nvme_submit_sync_cmd(dev->admin_q, (struct nvme_command *)&c,
								NULL, 0);
	if (ret)
		dev_err(dev->dev, "set bad block table failed (%d)\n", ret);
	return ret;
}

static inline void nvme_nvm_rqtocmd(struct request *rq, struct nvm_rq *rqd,
				struct nvme_ns *ns, struct nvme_nvm_command *c)
{
	c->ph_rw.opcode = rqd->opcode;
	c->ph_rw.nsid = cpu_to_le32(ns->ns_id);
	c->ph_rw.spba = cpu_to_le64(rqd->ppa_addr.ppa);
	c->ph_rw.control = cpu_to_le16(rqd->flags);
	c->ph_rw.length = cpu_to_le16(rqd->nr_pages - 1);

	if (rqd->opcode == NVM_OP_HBWRITE || rqd->opcode == NVM_OP_HBREAD)
		c->hb_rw.slba = cpu_to_le64(nvme_block_nr(ns,
						rqd->bio->bi_iter.bi_sector));
}

static void nvme_nvm_end_io(struct request *rq, int error)
{
	struct nvm_rq *rqd = rq->end_io_data;
	struct nvm_dev *dev = rqd->dev;

	if (dev->mt->end_io(rqd, error))
		pr_err("nvme: err status: %x result: %lx\n",
				rq->errors, (unsigned long)rq->special);

	kfree(rq->cmd);
	blk_mq_free_request(rq);
}

static int nvme_nvm_submit_io(struct request_queue *q, struct nvm_rq *rqd)
{
	struct nvme_ns *ns = q->queuedata;
	struct request *rq;
	struct bio *bio = rqd->bio;
	struct nvme_nvm_command *cmd;

	rq = blk_mq_alloc_request(q, bio_rw(bio), 0);
	if (IS_ERR(rq))
		return -ENOMEM;

	cmd = kzalloc(sizeof(struct nvme_nvm_command), GFP_KERNEL);
	if (!cmd) {
		blk_mq_free_request(rq);
		return -ENOMEM;
	}

	rq->cmd_type = REQ_TYPE_DRV_PRIV;
	rq->ioprio = bio_prio(bio);

	if (bio_has_data(bio))
		rq->nr_phys_segments = bio_phys_segments(q, bio);

	rq->__data_len = bio->bi_iter.bi_size;
	rq->bio = rq->biotail = bio;

	nvme_nvm_rqtocmd(rq, rqd, ns, cmd);

	rq->cmd = (unsigned char *)cmd;
	rq->cmd_len = sizeof(struct nvme_nvm_command);
	rq->special = (void *)0;

	rq->end_io_data = rqd;

	blk_execute_rq_nowait(q, NULL, rq, 0, nvme_nvm_end_io);

	return 0;
}

static int nvme_nvm_erase_block(struct request_queue *q, struct nvm_rq *rqd)
{
	struct nvme_ns *ns = q->queuedata;
	struct nvme_nvm_command c = {};

	c.erase.opcode = NVM_OP_ERASE;
	c.erase.nsid = cpu_to_le32(ns->ns_id);
	c.erase.spba = cpu_to_le64(rqd->ppa_addr.ppa);
	c.erase.length = cpu_to_le16(rqd->nr_pages - 1);

	return nvme_submit_sync_cmd(q, (struct nvme_command *)&c, NULL, 0);
}

static void *nvme_nvm_create_dma_pool(struct request_queue *q, char *name)
{
	struct nvme_ns *ns = q->queuedata;
	struct nvme_dev *dev = ns->dev;

	return dma_pool_create(name, dev->dev, PAGE_SIZE, PAGE_SIZE, 0);
}

static void nvme_nvm_destroy_dma_pool(void *pool)
{
	struct dma_pool *dma_pool = pool;

	dma_pool_destroy(dma_pool);
}

static void *nvme_nvm_dev_dma_alloc(struct request_queue *q, void *pool,
				    gfp_t mem_flags, dma_addr_t *dma_handler)
{
	return dma_pool_alloc(pool, mem_flags, dma_handler);
}

static void nvme_nvm_dev_dma_free(void *pool, void *ppa_list,
							dma_addr_t dma_handler)
{
	dma_pool_free(pool, ppa_list, dma_handler);
}

static struct nvm_dev_ops nvme_nvm_dev_ops = {
	.identity		= nvme_nvm_identity,

	.get_l2p_tbl		= nvme_nvm_get_l2p_tbl,

	.get_bb_tbl		= nvme_nvm_get_bb_tbl,
	.set_bb_tbl		= nvme_nvm_set_bb_tbl,

	.submit_io		= nvme_nvm_submit_io,
	.erase_block		= nvme_nvm_erase_block,

	.create_dma_pool	= nvme_nvm_create_dma_pool,
	.destroy_dma_pool	= nvme_nvm_destroy_dma_pool,
	.dev_dma_alloc		= nvme_nvm_dev_dma_alloc,
	.dev_dma_free		= nvme_nvm_dev_dma_free,

	.max_phys_sect		= 64,
};

int nvme_nvm_register(struct request_queue *q, char *disk_name)
{
	return nvm_register(q, disk_name, &nvme_nvm_dev_ops);
}

void nvme_nvm_unregister(struct request_queue *q, char *disk_name)
{
	nvm_unregister(disk_name);
}

int nvme_nvm_ns_supported(struct nvme_ns *ns, struct nvme_id_ns *id)
{
	struct nvme_dev *dev = ns->dev;
	struct pci_dev *pdev = to_pci_dev(dev->dev);

	/* QEMU NVMe simulator - PCI ID + Vendor specific bit */
	if (pdev->vendor == PCI_VENDOR_ID_INTEL && pdev->device == 0x5845 &&
							id->vs[0] == 0x1)
		return 1;

	/* CNEX Labs - PCI ID + Vendor specific bit */
	if (pdev->vendor == 0x1d1d && pdev->device == 0x2807 &&
							id->vs[0] == 0x1)
		return 1;

	return 0;
}
#else
int nvme_nvm_register(struct request_queue *q, char *disk_name)
{
	return 0;
}
void nvme_nvm_unregister(struct request_queue *q, char *disk_name) {};
int nvme_nvm_ns_supported(struct nvme_ns *ns, struct nvme_id_ns *id)
{
	return 0;
}
#endif /* CONFIG_NVM */
Commit	Line	Data
ca064085 MB	1	/*
	2	* nvme-lightnvm.c - LightNVM NVMe device
	3	*
	4	* Copyright (C) 2014-2015 IT University of Copenhagen
	5	* Initial release: Matias Bjorling <mb@lightnvm.io>
	6	*
	7	* This program is free software; you can redistribute it and/or
	8	* modify it under the terms of the GNU General Public License version
	9	* 2 as published by the Free Software Foundation.
	10	*
	11	* This program is distributed in the hope that it will be useful, but
	12	* WITHOUT ANY WARRANTY; without even the implied warranty of
	13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	* General Public License for more details.
	15	*
	16	* You should have received a copy of the GNU General Public License
	17	* along with this program; see the file COPYING. If not, write to
	18	* the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139,
	19	* USA.
	20	*
	21	*/
	22
	23	#include "nvme.h"
	24
	25	#ifdef CONFIG_NVM
	26
	27	#include <linux/nvme.h>
	28	#include <linux/bitops.h>
	29	#include <linux/lightnvm.h>
	30	#include <linux/vmalloc.h>
	31
	32	enum nvme_nvm_admin_opcode {
	33	nvme_nvm_admin_identity = 0xe2,
	34	nvme_nvm_admin_get_l2p_tbl = 0xea,
	35	nvme_nvm_admin_get_bb_tbl = 0xf2,
	36	nvme_nvm_admin_set_bb_tbl = 0xf1,
	37	};
	38
	39	struct nvme_nvm_hb_rw {
	40	__u8 opcode;
	41	__u8 flags;
	42	__u16 command_id;
	43	__le32 nsid;
	44	__u64 rsvd2;
	45	__le64 metadata;
	46	__le64 prp1;
	47	__le64 prp2;
	48	__le64 spba;
	49	__le16 length;
	50	__le16 control;
	51	__le32 dsmgmt;
	52	__le64 slba;
	53	};
	54
	55	struct nvme_nvm_ph_rw {
	56	__u8 opcode;
	57	__u8 flags;
	58	__u16 command_id;
	59	__le32 nsid;
	60	__u64 rsvd2;
	61	__le64 metadata;
	62	__le64 prp1;
	63	__le64 prp2;
	64	__le64 spba;
65	__le16 length;
66	__le16 control;
67	__le32 dsmgmt;
68	__le64 resv;
69	};
70
71	struct nvme_nvm_identity {
72	__u8 opcode;
73	__u8 flags;
74	__u16 command_id;
75	__le32 nsid;
76	__u64 rsvd[2];
77	__le64 prp1;
78	__le64 prp2;
79	__le32 chnl_off;
80	__u32 rsvd11[5];
81	};
82
83	struct nvme_nvm_l2ptbl {
84	__u8 opcode;
85	__u8 flags;
86	__u16 command_id;
87	__le32 nsid;
88	__le32 cdw2[4];
89	__le64 prp1;
90	__le64 prp2;
91	__le64 slba;
92	__le32 nlb;
93	__le16 cdw14[6];
94	};
95
11450469	96	struct nvme_nvm_getbbtbl {
ca064085 MB	97	__u8 opcode;
	98	__u8 flags;
	99	__u16 command_id;
	100	__le32 nsid;
	101	__u64 rsvd[2];
	102	__le64 prp1;
	103	__le64 prp2;
11450469 MB	104	__le64 spba;
	105	__u32 rsvd4[4];
	106	};
	107
	108	struct nvme_nvm_setbbtbl {
	109	__u8 opcode;
	110	__u8 flags;
	111	__u16 command_id;
	112	__le32 nsid;
	113	__le64 rsvd[2];
	114	__le64 prp1;
	115	__le64 prp2;
	116	__le64 spba;
	117	__le16 nlb;
	118	__u8 value;
	119	__u8 rsvd3;
	120	__u32 rsvd4[3];
ca064085 MB	121	};
	122
	123	struct nvme_nvm_erase_blk {
	124	__u8 opcode;
	125	__u8 flags;
	126	__u16 command_id;
	127	__le32 nsid;
	128	__u64 rsvd[2];
	129	__le64 prp1;
	130	__le64 prp2;
	131	__le64 spba;
	132	__le16 length;
	133	__le16 control;
	134	__le32 dsmgmt;
	135	__le64 resv;
	136	};
	137
	138	struct nvme_nvm_command {
	139	union {
	140	struct nvme_common_command common;
	141	struct nvme_nvm_identity identity;
	142	struct nvme_nvm_hb_rw hb_rw;
	143	struct nvme_nvm_ph_rw ph_rw;
	144	struct nvme_nvm_l2ptbl l2p;
11450469 MB	145	struct nvme_nvm_getbbtbl get_bb;
11450469 MB	146	struct nvme_nvm_setbbtbl set_bb;
ca064085 MB	147	struct nvme_nvm_erase_blk erase;
	148	};
	149	};
	150
	151	struct nvme_nvm_id_group {
	152	__u8 mtype;
	153	__u8 fmtype;
	154	__le16 res16;
	155	__u8 num_ch;
	156	__u8 num_lun;
	157	__u8 num_pln;
36d5dbc6	158	__u8 rsvd1;
ca064085 MB	159	__le16 num_blk;
	160	__le16 num_pg;
	161	__le16 fpg_sz;
	162	__le16 csecs;
	163	__le16 sos;
36d5dbc6	164	__le16 rsvd2;
ca064085 MB	165	__le32 trdt;
	166	__le32 trdm;
	167	__le32 tprt;
	168	__le32 tprm;
	169	__le32 tbet;
	170	__le32 tbem;
	171	__le32 mpos;
12be5edf	172	__le32 mccap;
ca064085	173	__le16 cpar;
12be5edf	174	__u8 reserved[906];
ca064085 MB	175	} __packed;
	176
	177	struct nvme_nvm_addr_format {
	178	__u8 ch_offset;
	179	__u8 ch_len;
	180	__u8 lun_offset;
	181	__u8 lun_len;
	182	__u8 pln_offset;
	183	__u8 pln_len;
	184	__u8 blk_offset;
	185	__u8 blk_len;
	186	__u8 pg_offset;
	187	__u8 pg_len;
	188	__u8 sect_offset;
	189	__u8 sect_len;
	190	__u8 res[4];
	191	} __packed;
	192
	193	struct nvme_nvm_id {
	194	__u8 ver_id;
	195	__u8 vmnt;
	196	__u8 cgrps;
dad1b009	197	__u8 res;
ca064085 MB	198	__le32 cap;
	199	__le32 dom;
	200	struct nvme_nvm_addr_format ppaf;
dad1b009	201	__u8 resv[228];
ca064085 MB	202	struct nvme_nvm_id_group groups[4];
	203	} __packed;
	204
11450469 MB	205	struct nvme_nvm_bb_tbl {
	206	__u8 tblid[4];
	207	__le16 verid;
	208	__le16 revid;
	209	__le32 rvsd1;
	210	__le32 tblks;
	211	__le32 tfact;
	212	__le32 tgrown;
	213	__le32 tdresv;
	214	__le32 thresv;
	215	__le32 rsvd2[8];
	216	__u8 blk[0];
	217	};
	218
ca064085 MB	219	/*
	220	* Check we didn't inadvertently grow the command struct
	221	*/
	222	static inline void _nvme_nvm_check_size(void)
	223	{
	224	BUILD_BUG_ON(sizeof(struct nvme_nvm_identity) != 64);
	225	BUILD_BUG_ON(sizeof(struct nvme_nvm_hb_rw) != 64);
	226	BUILD_BUG_ON(sizeof(struct nvme_nvm_ph_rw) != 64);
11450469 MB	227	BUILD_BUG_ON(sizeof(struct nvme_nvm_getbbtbl) != 64);
11450469 MB	228	BUILD_BUG_ON(sizeof(struct nvme_nvm_setbbtbl) != 64);
ca064085 MB	229	BUILD_BUG_ON(sizeof(struct nvme_nvm_l2ptbl) != 64);
	230	BUILD_BUG_ON(sizeof(struct nvme_nvm_erase_blk) != 64);
	231	BUILD_BUG_ON(sizeof(struct nvme_nvm_id_group) != 960);
	232	BUILD_BUG_ON(sizeof(struct nvme_nvm_addr_format) != 128);
	233	BUILD_BUG_ON(sizeof(struct nvme_nvm_id) != 4096);
11450469	234	BUILD_BUG_ON(sizeof(struct nvme_nvm_bb_tbl) != 512);
ca064085 MB	235	}
	236
	237	static int init_grps(struct nvm_id nvm_id, struct nvme_nvm_id nvme_nvm_id)
	238	{
	239	struct nvme_nvm_id_group *src;
	240	struct nvm_id_group *dst;
	241	int i, end;
	242
	243	end = min_t(u32, 4, nvm_id->cgrps);
	244
	245	for (i = 0; i < end; i++) {
	246	src = &nvme_nvm_id->groups[i];
	247	dst = &nvm_id->groups[i];
	248
	249	dst->mtype = src->mtype;
	250	dst->fmtype = src->fmtype;
	251	dst->num_ch = src->num_ch;
	252	dst->num_lun = src->num_lun;
	253	dst->num_pln = src->num_pln;
	254
	255	dst->num_pg = le16_to_cpu(src->num_pg);
	256	dst->num_blk = le16_to_cpu(src->num_blk);
	257	dst->fpg_sz = le16_to_cpu(src->fpg_sz);
	258	dst->csecs = le16_to_cpu(src->csecs);
	259	dst->sos = le16_to_cpu(src->sos);
	260
	261	dst->trdt = le32_to_cpu(src->trdt);
	262	dst->trdm = le32_to_cpu(src->trdm);
	263	dst->tprt = le32_to_cpu(src->tprt);
	264	dst->tprm = le32_to_cpu(src->tprm);
	265	dst->tbet = le32_to_cpu(src->tbet);
	266	dst->tbem = le32_to_cpu(src->tbem);
	267	dst->mpos = le32_to_cpu(src->mpos);
12be5edf	268	dst->mccap = le32_to_cpu(src->mccap);
ca064085 MB	269
	270	dst->cpar = le16_to_cpu(src->cpar);
	271	}
	272
	273	return 0;
	274	}
	275
	276	static int nvme_nvm_identity(struct request_queue q, struct nvm_id nvm_id)
	277	{
	278	struct nvme_ns *ns = q->queuedata;
47b3115a	279	struct nvme_dev *dev = ns->dev;
ca064085 MB	280	struct nvme_nvm_id *nvme_nvm_id;
	281	struct nvme_nvm_command c = {};
	282	int ret;
	283
	284	c.identity.opcode = nvme_nvm_admin_identity;
	285	c.identity.nsid = cpu_to_le32(ns->ns_id);
	286	c.identity.chnl_off = 0;
	287
	288	nvme_nvm_id = kmalloc(sizeof(struct nvme_nvm_id), GFP_KERNEL);
	289	if (!nvme_nvm_id)
	290	return -ENOMEM;
	291
47b3115a WT	292	ret = nvme_submit_sync_cmd(dev->admin_q, (struct nvme_command *)&c,
47b3115a WT	293	nvme_nvm_id, sizeof(struct nvme_nvm_id));
ca064085 MB	294	if (ret) {
	295	ret = -EIO;
	296	goto out;
	297	}
	298
	299	nvm_id->ver_id = nvme_nvm_id->ver_id;
	300	nvm_id->vmnt = nvme_nvm_id->vmnt;
	301	nvm_id->cgrps = nvme_nvm_id->cgrps;
	302	nvm_id->cap = le32_to_cpu(nvme_nvm_id->cap);
	303	nvm_id->dom = le32_to_cpu(nvme_nvm_id->dom);
2393bd39 MB	304	memcpy(&nvm_id->ppaf, &nvme_nvm_id->ppaf,
2393bd39 MB	305	sizeof(struct nvme_nvm_addr_format));
ca064085 MB	306
	307	ret = init_grps(nvm_id, nvme_nvm_id);
	308	out:
	309	kfree(nvme_nvm_id);
	310	return ret;
	311	}
	312
	313	static int nvme_nvm_get_l2p_tbl(struct request_queue *q, u64 slba, u32 nlb,
	314	nvm_l2p_update_fn update_l2p, void priv)
	315	{
	316	struct nvme_ns *ns = q->queuedata;
	317	struct nvme_dev *dev = ns->dev;
	318	struct nvme_nvm_command c = {};
47b3115a	319	u32 len = queue_max_hw_sectors(dev->admin_q) << 9;
5f436e5e	320	u32 nlb_pr_rq = len / sizeof(u64);
ca064085 MB	321	u64 cmd_slba = slba;
	322	void *entries;
	323	int ret = 0;
	324
	325	c.l2p.opcode = nvme_nvm_admin_get_l2p_tbl;
	326	c.l2p.nsid = cpu_to_le32(ns->ns_id);
	327	entries = kmalloc(len, GFP_KERNEL);
	328	if (!entries)
	329	return -ENOMEM;
	330
	331	while (nlb) {
5f436e5e	332	u32 cmd_nlb = min(nlb_pr_rq, nlb);
ca064085 MB	333
	334	c.l2p.slba = cpu_to_le64(cmd_slba);
	335	c.l2p.nlb = cpu_to_le32(cmd_nlb);
	336
47b3115a WT	337	ret = nvme_submit_sync_cmd(dev->admin_q,
47b3115a WT	338	(struct nvme_command *)&c, entries, len);
ca064085 MB	339	if (ret) {
	340	dev_err(dev->dev, "L2P table transfer failed (%d)\n",
	341	ret);
	342	ret = -EIO;
	343	goto out;
	344	}
	345
	346	if (update_l2p(cmd_slba, cmd_nlb, entries, priv)) {
	347	ret = -EINTR;
	348	goto out;
	349	}
	350
	351	cmd_slba += cmd_nlb;
	352	nlb -= cmd_nlb;
	353	}
	354
	355	out:
	356	kfree(entries);
	357	return ret;
	358	}
	359
11450469 MB	360	static int nvme_nvm_get_bb_tbl(struct request_queue *q, struct ppa_addr ppa,
	361	int nr_blocks, nvm_bb_update_fn *update_bbtbl,
	362	void *priv)
ca064085 MB	363	{
	364	struct nvme_ns *ns = q->queuedata;
	365	struct nvme_dev *dev = ns->dev;
	366	struct nvme_nvm_command c = {};
11450469 MB	367	struct nvme_nvm_bb_tbl *bb_tbl;
11450469 MB	368	int tblsz = sizeof(struct nvme_nvm_bb_tbl) + nr_blocks;
ca064085 MB	369	int ret = 0;
	370
	371	c.get_bb.opcode = nvme_nvm_admin_get_bb_tbl;
	372	c.get_bb.nsid = cpu_to_le32(ns->ns_id);
11450469	373	c.get_bb.spba = cpu_to_le64(ppa.ppa);
ca064085	374
11450469 MB	375	bb_tbl = kzalloc(tblsz, GFP_KERNEL);
	376	if (!bb_tbl)
	377	return -ENOMEM;
ca064085	378
47b3115a WT	379	ret = nvme_submit_sync_cmd(dev->admin_q, (struct nvme_command *)&c,
47b3115a WT	380	bb_tbl, tblsz);
ca064085 MB	381	if (ret) {
	382	dev_err(dev->dev, "get bad block table failed (%d)\n", ret);
	383	ret = -EIO;
	384	goto out;
	385	}
	386
11450469 MB	387	if (bb_tbl->tblid[0] != 'B' \|\| bb_tbl->tblid[1] != 'B' \|\|
	388	bb_tbl->tblid[2] != 'L' \|\| bb_tbl->tblid[3] != 'T') {
	389	dev_err(dev->dev, "bbt format mismatch\n");
	390	ret = -EINVAL;
	391	goto out;
	392	}
	393
	394	if (le16_to_cpu(bb_tbl->verid) != 1) {
	395	ret = -EINVAL;
	396	dev_err(dev->dev, "bbt version not supported\n");
	397	goto out;
	398	}
	399
	400	if (le32_to_cpu(bb_tbl->tblks) != nr_blocks) {
	401	ret = -EINVAL;
	402	dev_err(dev->dev, "bbt unsuspected blocks returned (%u!=%u)",
	403	le32_to_cpu(bb_tbl->tblks), nr_blocks);
	404	goto out;
	405	}
	406
	407	ret = update_bbtbl(ppa, nr_blocks, bb_tbl->blk, priv);
ca064085 MB	408	if (ret) {
	409	ret = -EINTR;
	410	goto out;
	411	}
	412
	413	out:
11450469 MB	414	kfree(bb_tbl);
	415	return ret;
	416	}
	417
	418	static int nvme_nvm_set_bb_tbl(struct request_queue q, struct nvm_rq rqd,
	419	int type)
	420	{
	421	struct nvme_ns *ns = q->queuedata;
	422	struct nvme_dev *dev = ns->dev;
	423	struct nvme_nvm_command c = {};
	424	int ret = 0;
	425
	426	c.set_bb.opcode = nvme_nvm_admin_set_bb_tbl;
	427	c.set_bb.nsid = cpu_to_le32(ns->ns_id);
	428	c.set_bb.spba = cpu_to_le64(rqd->ppa_addr.ppa);
	429	c.set_bb.nlb = cpu_to_le16(rqd->nr_pages - 1);
	430	c.set_bb.value = type;
	431
47b3115a WT	432	ret = nvme_submit_sync_cmd(dev->admin_q, (struct nvme_command *)&c,
47b3115a WT	433	NULL, 0);
11450469 MB	434	if (ret)
11450469 MB	435	dev_err(dev->dev, "set bad block table failed (%d)\n", ret);
ca064085 MB	436	return ret;
	437	}
	438
	439	static inline void nvme_nvm_rqtocmd(struct request rq, struct nvm_rq rqd,
	440	struct nvme_ns ns, struct nvme_nvm_command c)
	441	{
	442	c->ph_rw.opcode = rqd->opcode;
	443	c->ph_rw.nsid = cpu_to_le32(ns->ns_id);
	444	c->ph_rw.spba = cpu_to_le64(rqd->ppa_addr.ppa);
	445	c->ph_rw.control = cpu_to_le16(rqd->flags);
	446	c->ph_rw.length = cpu_to_le16(rqd->nr_pages - 1);
	447
	448	if (rqd->opcode == NVM_OP_HBWRITE \|\| rqd->opcode == NVM_OP_HBREAD)
	449	c->hb_rw.slba = cpu_to_le64(nvme_block_nr(ns,
	450	rqd->bio->bi_iter.bi_sector));
	451	}
	452
	453	static void nvme_nvm_end_io(struct request *rq, int error)
	454	{
	455	struct nvm_rq *rqd = rq->end_io_data;
	456	struct nvm_dev *dev = rqd->dev;
	457
	458	if (dev->mt->end_io(rqd, error))
	459	pr_err("nvme: err status: %x result: %lx\n",
	460	rq->errors, (unsigned long)rq->special);
	461
	462	kfree(rq->cmd);
	463	blk_mq_free_request(rq);
	464	}
	465
	466	static int nvme_nvm_submit_io(struct request_queue q, struct nvm_rq rqd)
	467	{
	468	struct nvme_ns *ns = q->queuedata;
	469	struct request *rq;
	470	struct bio *bio = rqd->bio;
	471	struct nvme_nvm_command *cmd;
	472
6f3b0e8b	473	rq = blk_mq_alloc_request(q, bio_rw(bio), 0);
ca064085 MB	474	if (IS_ERR(rq))
	475	return -ENOMEM;
	476
	477	cmd = kzalloc(sizeof(struct nvme_nvm_command), GFP_KERNEL);
	478	if (!cmd) {
	479	blk_mq_free_request(rq);
	480	return -ENOMEM;
	481	}
	482
	483	rq->cmd_type = REQ_TYPE_DRV_PRIV;
	484	rq->ioprio = bio_prio(bio);
	485
	486	if (bio_has_data(bio))
	487	rq->nr_phys_segments = bio_phys_segments(q, bio);
	488
	489	rq->__data_len = bio->bi_iter.bi_size;
	490	rq->bio = rq->biotail = bio;
	491
	492	nvme_nvm_rqtocmd(rq, rqd, ns, cmd);
	493
	494	rq->cmd = (unsigned char *)cmd;
	495	rq->cmd_len = sizeof(struct nvme_nvm_command);
	496	rq->special = (void *)0;
	497
	498	rq->end_io_data = rqd;
	499
	500	blk_execute_rq_nowait(q, NULL, rq, 0, nvme_nvm_end_io);
	501
	502	return 0;
	503	}
	504
	505	static int nvme_nvm_erase_block(struct request_queue q, struct nvm_rq rqd)
	506	{
	507	struct nvme_ns *ns = q->queuedata;
	508	struct nvme_nvm_command c = {};
	509
	510	c.erase.opcode = NVM_OP_ERASE;
	511	c.erase.nsid = cpu_to_le32(ns->ns_id);
	512	c.erase.spba = cpu_to_le64(rqd->ppa_addr.ppa);
	513	c.erase.length = cpu_to_le16(rqd->nr_pages - 1);
	514
	515	return nvme_submit_sync_cmd(q, (struct nvme_command *)&c, NULL, 0);
	516	}
	517
	518	static void nvme_nvm_create_dma_pool(struct request_queue q, char *name)
	519	{
	520	struct nvme_ns *ns = q->queuedata;
	521	struct nvme_dev *dev = ns->dev;
	522
	523	return dma_pool_create(name, dev->dev, PAGE_SIZE, PAGE_SIZE, 0);
	524	}
	525
	526	static void nvme_nvm_destroy_dma_pool(void *pool)
	527	{
	528	struct dma_pool *dma_pool = pool;
	529
	530	dma_pool_destroy(dma_pool);
	531	}
	532
	533	static void nvme_nvm_dev_dma_alloc(struct request_queue q, void *pool,
	534	gfp_t mem_flags, dma_addr_t *dma_handler)
	535	{
	536	return dma_pool_alloc(pool, mem_flags, dma_handler);
	537	}
538
539	static void nvme_nvm_dev_dma_free(void pool, void ppa_list,
540	dma_addr_t dma_handler)
541	{
542	dma_pool_free(pool, ppa_list, dma_handler);
543	}
544
545	static struct nvm_dev_ops nvme_nvm_dev_ops = {
546	.identity = nvme_nvm_identity,
547
548	.get_l2p_tbl = nvme_nvm_get_l2p_tbl,
549
550	.get_bb_tbl = nvme_nvm_get_bb_tbl,
11450469	551	.set_bb_tbl = nvme_nvm_set_bb_tbl,
ca064085 MB	552
	553	.submit_io = nvme_nvm_submit_io,
	554	.erase_block = nvme_nvm_erase_block,
	555
	556	.create_dma_pool = nvme_nvm_create_dma_pool,
	557	.destroy_dma_pool = nvme_nvm_destroy_dma_pool,
	558	.dev_dma_alloc = nvme_nvm_dev_dma_alloc,
	559	.dev_dma_free = nvme_nvm_dev_dma_free,
	560
	561	.max_phys_sect = 64,
	562	};
	563
	564	int nvme_nvm_register(struct request_queue q, char disk_name)
	565	{
	566	return nvm_register(q, disk_name, &nvme_nvm_dev_ops);
	567	}
	568
	569	void nvme_nvm_unregister(struct request_queue q, char disk_name)
	570	{
	571	nvm_unregister(disk_name);
	572	}
	573
	574	int nvme_nvm_ns_supported(struct nvme_ns ns, struct nvme_id_ns id)
	575	{
	576	struct nvme_dev *dev = ns->dev;
	577	struct pci_dev *pdev = to_pci_dev(dev->dev);
	578
	579	/* QEMU NVMe simulator - PCI ID + Vendor specific bit */
	580	if (pdev->vendor == PCI_VENDOR_ID_INTEL && pdev->device == 0x5845 &&
	581	id->vs[0] == 0x1)
	582	return 1;
	583
	584	/* CNEX Labs - PCI ID + Vendor specific bit */
	585	if (pdev->vendor == 0x1d1d && pdev->device == 0x2807 &&
	586	id->vs[0] == 0x1)
	587	return 1;
	588
	589	return 0;
	590	}
	591	#else
	592	int nvme_nvm_register(struct request_queue q, char disk_name)
	593	{
	594	return 0;
	595	}
	596	void nvme_nvm_unregister(struct request_queue q, char disk_name) {};
	597	int nvme_nvm_ns_supported(struct nvme_ns ns, struct nvme_id_ns id)
	598	{
	599	return 0;
	600	}
	601	#endif /* CONFIG_NVM */