[mirror_ubuntu-kernels.git] / drivers / nvme / host / nvme.h

/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Copyright (c) 2011-2014, Intel Corporation.
 */

#ifndef _NVME_H
#define _NVME_H

#include <linux/nvme.h>
#include <linux/cdev.h>
#include <linux/pci.h>
#include <linux/kref.h>
#include <linux/blk-mq.h>
#include <linux/lightnvm.h>
#include <linux/sed-opal.h>
#include <linux/fault-inject.h>
#include <linux/rcupdate.h>

extern unsigned int nvme_io_timeout;
#define NVME_IO_TIMEOUT	(nvme_io_timeout * HZ)

extern unsigned int admin_timeout;
#define ADMIN_TIMEOUT	(admin_timeout * HZ)

#define NVME_DEFAULT_KATO	5
#define NVME_KATO_GRACE		10

extern struct workqueue_struct *nvme_wq;
extern struct workqueue_struct *nvme_reset_wq;
extern struct workqueue_struct *nvme_delete_wq;

enum {
	NVME_NS_LBA		= 0,
	NVME_NS_LIGHTNVM	= 1,
};

/*
 * List of workarounds for devices that required behavior not specified in
 * the standard.
 */
enum nvme_quirks {
	/*
	 * Prefers I/O aligned to a stripe size specified in a vendor
	 * specific Identify field.
	 */
	NVME_QUIRK_STRIPE_SIZE			= (1 << 0),

	/*
	 * The controller doesn't handle Identify value others than 0 or 1
	 * correctly.
	 */
	NVME_QUIRK_IDENTIFY_CNS			= (1 << 1),

	/*
	 * The controller deterministically returns O's on reads to
	 * logical blocks that deallocate was called on.
	 */
	NVME_QUIRK_DEALLOCATE_ZEROES		= (1 << 2),

	/*
	 * The controller needs a delay before starts checking the device
	 * readiness, which is done by reading the NVME_CSTS_RDY bit.
	 */
	NVME_QUIRK_DELAY_BEFORE_CHK_RDY		= (1 << 3),

	/*
	 * APST should not be used.
	 */
	NVME_QUIRK_NO_APST			= (1 << 4),

	/*
	 * The deepest sleep state should not be used.
	 */
	NVME_QUIRK_NO_DEEPEST_PS		= (1 << 5),

	/*
	 * Supports the LighNVM command set if indicated in vs[1].
	 */
	NVME_QUIRK_LIGHTNVM			= (1 << 6),

	/*
	 * Set MEDIUM priority on SQ creation
	 */
	NVME_QUIRK_MEDIUM_PRIO_SQ		= (1 << 7),

	/*
	 * Ignore device provided subnqn.
	 */
	NVME_QUIRK_IGNORE_DEV_SUBNQN		= (1 << 8),

	/*
	 * Broken Write Zeroes.
	 */
	NVME_QUIRK_DISABLE_WRITE_ZEROES		= (1 << 9),
};

/*
 * Common request structure for NVMe passthrough.  All drivers must have
 * this structure as the first member of their request-private data.
 */
struct nvme_request {
	struct nvme_command	*cmd;
	union nvme_result	result;
	u8			retries;
	u8			flags;
	u16			status;
	struct nvme_ctrl	*ctrl;
};

/*
 * Mark a bio as coming in through the mpath node.
 */
#define REQ_NVME_MPATH		REQ_DRV

enum {
	NVME_REQ_CANCELLED		= (1 << 0),
	NVME_REQ_USERCMD		= (1 << 1),
};

static inline struct nvme_request *nvme_req(struct request *req)
{
	return blk_mq_rq_to_pdu(req);
}

static inline u16 nvme_req_qid(struct request *req)
{
	if (!req->rq_disk)
		return 0;
	return blk_mq_unique_tag_to_hwq(blk_mq_unique_tag(req)) + 1;
}

/* The below value is the specific amount of delay needed before checking
 * readiness in case of the PCI_DEVICE(0x1c58, 0x0003), which needs the
 * NVME_QUIRK_DELAY_BEFORE_CHK_RDY quirk enabled. The value (in ms) was
 * found empirically.
 */
#define NVME_QUIRK_DELAY_AMOUNT		2300

enum nvme_ctrl_state {
	NVME_CTRL_NEW,
	NVME_CTRL_LIVE,
	NVME_CTRL_ADMIN_ONLY,    /* Only admin queue live */
	NVME_CTRL_RESETTING,
	NVME_CTRL_CONNECTING,
	NVME_CTRL_DELETING,
	NVME_CTRL_DEAD,
};

struct nvme_ctrl {
	bool comp_seen;
	enum nvme_ctrl_state state;
	bool identified;
	spinlock_t lock;
	struct mutex scan_lock;
	const struct nvme_ctrl_ops *ops;
	struct request_queue *admin_q;
	struct request_queue *connect_q;
	struct device *dev;
	int instance;
	int numa_node;
	struct blk_mq_tag_set *tagset;
	struct blk_mq_tag_set *admin_tagset;
	struct list_head namespaces;
	struct rw_semaphore namespaces_rwsem;
	struct device ctrl_device;
	struct device *device;	/* char device */
	struct cdev cdev;
	struct work_struct reset_work;
	struct work_struct delete_work;

	struct nvme_subsystem *subsys;
	struct list_head subsys_entry;

	struct opal_dev *opal_dev;

	char name[12];
	u16 cntlid;

	u32 ctrl_config;
	u16 mtfa;
	u32 queue_count;

	u64 cap;
	u32 page_size;
	u32 max_hw_sectors;
	u32 max_segments;
	u16 crdt[3];
	u16 oncs;
	u16 oacs;
	u16 nssa;
	u16 nr_streams;
	u32 max_namespaces;
	atomic_t abort_limit;
	u8 vwc;
	u32 vs;
	u32 sgls;
	u16 kas;
	u8 npss;
	u8 apsta;
	u32 oaes;
	u32 aen_result;
	u32 ctratt;
	unsigned int shutdown_timeout;
	unsigned int kato;
	bool subsystem;
	unsigned long quirks;
	struct nvme_id_power_state psd[32];
	struct nvme_effects_log *effects;
	struct work_struct scan_work;
	struct work_struct async_event_work;
	struct delayed_work ka_work;
	struct nvme_command ka_cmd;
	struct work_struct fw_act_work;
	unsigned long events;

#ifdef CONFIG_NVME_MULTIPATH
	/* asymmetric namespace access: */
	u8 anacap;
	u8 anatt;
	u32 anagrpmax;
	u32 nanagrpid;
	struct mutex ana_lock;
	struct nvme_ana_rsp_hdr *ana_log_buf;
	size_t ana_log_size;
	struct timer_list anatt_timer;
	struct work_struct ana_work;
#endif

	/* Power saving configuration */
	u64 ps_max_latency_us;
	bool apst_enabled;

	/* PCIe only: */
	u32 hmpre;
	u32 hmmin;
	u32 hmminds;
	u16 hmmaxd;

	/* Fabrics only */
	u16 sqsize;
	u32 ioccsz;
	u32 iorcsz;
	u16 icdoff;
	u16 maxcmd;
	int nr_reconnects;
	struct nvmf_ctrl_options *opts;

	struct page *discard_page;
	unsigned long discard_page_busy;
};

enum nvme_iopolicy {
	NVME_IOPOLICY_NUMA,
	NVME_IOPOLICY_RR,
};

struct nvme_subsystem {
	int			instance;
	struct device		dev;
	/*
	 * Because we unregister the device on the last put we need
	 * a separate refcount.
	 */
	struct kref		ref;
	struct list_head	entry;
	struct mutex		lock;
	struct list_head	ctrls;
	struct list_head	nsheads;
	char			subnqn[NVMF_NQN_SIZE];
	char			serial[20];
	char			model[40];
	char			firmware_rev[8];
	u8			cmic;
	u16			vendor_id;
	struct ida		ns_ida;
#ifdef CONFIG_NVME_MULTIPATH
	enum nvme_iopolicy	iopolicy;
#endif
};

/*
 * Container structure for uniqueue namespace identifiers.
 */
struct nvme_ns_ids {
	u8	eui64[8];
	u8	nguid[16];
	uuid_t	uuid;
};

/*
 * Anchor structure for namespaces.  There is one for each namespace in a
 * NVMe subsystem that any of our controllers can see, and the namespace
 * structure for each controller is chained of it.  For private namespaces
 * there is a 1:1 relation to our namespace structures, that is ->list
 * only ever has a single entry for private namespaces.
 */
struct nvme_ns_head {
	struct list_head	list;
	struct srcu_struct      srcu;
	struct nvme_subsystem	*subsys;
	unsigned		ns_id;
	struct nvme_ns_ids	ids;
	struct list_head	entry;
	struct kref		ref;
	int			instance;
#ifdef CONFIG_NVME_MULTIPATH
	struct gendisk		*disk;
	struct bio_list		requeue_list;
	spinlock_t		requeue_lock;
	struct work_struct	requeue_work;
	struct mutex		lock;
	struct nvme_ns __rcu	*current_path[];
#endif
};

#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
struct nvme_fault_inject {
	struct fault_attr attr;
	struct dentry *parent;
	bool dont_retry;	/* DNR, do not retry */
	u16 status;		/* status code */
};
#endif

struct nvme_ns {
	struct list_head list;

	struct nvme_ctrl *ctrl;
	struct request_queue *queue;
	struct gendisk *disk;
#ifdef CONFIG_NVME_MULTIPATH
	enum nvme_ana_state ana_state;
	u32 ana_grpid;
#endif
	struct list_head siblings;
	struct nvm_dev *ndev;
	struct kref kref;
	struct nvme_ns_head *head;

	int lba_shift;
	u16 ms;
	u16 sgs;
	u32 sws;
	bool ext;
	u8 pi_type;
	unsigned long flags;
#define NVME_NS_REMOVING	0
#define NVME_NS_DEAD     	1
#define NVME_NS_ANA_PENDING	2
	u16 noiob;

#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
	struct nvme_fault_inject fault_inject;
#endif

};

struct nvme_ctrl_ops {
	const char *name;
	struct module *module;
	unsigned int flags;
#define NVME_F_FABRICS			(1 << 0)
#define NVME_F_METADATA_SUPPORTED	(1 << 1)
#define NVME_F_PCI_P2PDMA		(1 << 2)
	int (*reg_read32)(struct nvme_ctrl *ctrl, u32 off, u32 *val);
	int (*reg_write32)(struct nvme_ctrl *ctrl, u32 off, u32 val);
	int (*reg_read64)(struct nvme_ctrl *ctrl, u32 off, u64 *val);
	void (*free_ctrl)(struct nvme_ctrl *ctrl);
	void (*submit_async_event)(struct nvme_ctrl *ctrl);
	void (*delete_ctrl)(struct nvme_ctrl *ctrl);
	int (*get_address)(struct nvme_ctrl *ctrl, char *buf, int size);
};

#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
void nvme_fault_inject_init(struct nvme_ns *ns);
void nvme_fault_inject_fini(struct nvme_ns *ns);
void nvme_should_fail(struct request *req);
#else
static inline void nvme_fault_inject_init(struct nvme_ns *ns) {}
static inline void nvme_fault_inject_fini(struct nvme_ns *ns) {}
static inline void nvme_should_fail(struct request *req) {}
#endif

static inline int nvme_reset_subsystem(struct nvme_ctrl *ctrl)
{
	if (!ctrl->subsystem)
		return -ENOTTY;
	return ctrl->ops->reg_write32(ctrl, NVME_REG_NSSR, 0x4E564D65);
}

static inline u64 nvme_block_nr(struct nvme_ns *ns, sector_t sector)
{
	return (sector >> (ns->lba_shift - 9));
}

static inline void nvme_end_request(struct request *req, __le16 status,
		union nvme_result result)
{
	struct nvme_request *rq = nvme_req(req);

	rq->status = le16_to_cpu(status) >> 1;
	rq->result = result;
	/* inject error when permitted by fault injection framework */
	nvme_should_fail(req);
	blk_mq_complete_request(req);
}

static inline void nvme_get_ctrl(struct nvme_ctrl *ctrl)
{
	get_device(ctrl->device);
}

static inline void nvme_put_ctrl(struct nvme_ctrl *ctrl)
{
	put_device(ctrl->device);
}

void nvme_complete_rq(struct request *req);
bool nvme_cancel_request(struct request *req, void *data, bool reserved);
bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
		enum nvme_ctrl_state new_state);
int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap);
int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap);
int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl);
int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
		const struct nvme_ctrl_ops *ops, unsigned long quirks);
void nvme_uninit_ctrl(struct nvme_ctrl *ctrl);
void nvme_start_ctrl(struct nvme_ctrl *ctrl);
void nvme_stop_ctrl(struct nvme_ctrl *ctrl);
void nvme_put_ctrl(struct nvme_ctrl *ctrl);
int nvme_init_identify(struct nvme_ctrl *ctrl);

void nvme_remove_namespaces(struct nvme_ctrl *ctrl);

int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,
		bool send);

void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
		volatile union nvme_result *res);

void nvme_stop_queues(struct nvme_ctrl *ctrl);
void nvme_start_queues(struct nvme_ctrl *ctrl);
void nvme_kill_queues(struct nvme_ctrl *ctrl);
void nvme_unfreeze(struct nvme_ctrl *ctrl);
void nvme_wait_freeze(struct nvme_ctrl *ctrl);
void nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout);
void nvme_start_freeze(struct nvme_ctrl *ctrl);

#define NVME_QID_ANY -1
struct request *nvme_alloc_request(struct request_queue *q,
		struct nvme_command *cmd, blk_mq_req_flags_t flags, int qid);
void nvme_cleanup_cmd(struct request *req);
blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req,
		struct nvme_command *cmd);
int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
		void *buf, unsigned bufflen);
int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
		union nvme_result *result, void *buffer, unsigned bufflen,
		unsigned timeout, int qid, int at_head,
		blk_mq_req_flags_t flags, bool poll);
int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count);
void nvme_stop_keep_alive(struct nvme_ctrl *ctrl);
int nvme_reset_ctrl(struct nvme_ctrl *ctrl);
int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl);
int nvme_delete_ctrl(struct nvme_ctrl *ctrl);

int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp,
		void *log, size_t size, u64 offset);

extern const struct attribute_group *nvme_ns_id_attr_groups[];
extern const struct block_device_operations nvme_ns_head_ops;

#ifdef CONFIG_NVME_MULTIPATH
bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl);
void nvme_set_disk_name(char *disk_name, struct nvme_ns *ns,
			struct nvme_ctrl *ctrl, int *flags);
void nvme_failover_req(struct request *req);
void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl);
int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl,struct nvme_ns_head *head);
void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id);
void nvme_mpath_remove_disk(struct nvme_ns_head *head);
int nvme_mpath_init(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id);
void nvme_mpath_uninit(struct nvme_ctrl *ctrl);
void nvme_mpath_stop(struct nvme_ctrl *ctrl);
void nvme_mpath_clear_current_path(struct nvme_ns *ns);
struct nvme_ns *nvme_find_path(struct nvme_ns_head *head);

static inline void nvme_mpath_check_last_path(struct nvme_ns *ns)
{
	struct nvme_ns_head *head = ns->head;

	if (head->disk && list_empty(&head->list))
		kblockd_schedule_work(&head->requeue_work);
}

extern struct device_attribute dev_attr_ana_grpid;
extern struct device_attribute dev_attr_ana_state;
extern struct device_attribute subsys_attr_iopolicy;

#else
static inline bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl)
{
	return false;
}
/*
 * Without the multipath code enabled, multiple controller per subsystems are
 * visible as devices and thus we cannot use the subsystem instance.
 */
static inline void nvme_set_disk_name(char *disk_name, struct nvme_ns *ns,
				      struct nvme_ctrl *ctrl, int *flags)
{
	sprintf(disk_name, "nvme%dn%d", ctrl->instance, ns->head->instance);
}

static inline void nvme_failover_req(struct request *req)
{
}
static inline void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)
{
}
static inline int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl,
		struct nvme_ns_head *head)
{
	return 0;
}
static inline void nvme_mpath_add_disk(struct nvme_ns *ns,
		struct nvme_id_ns *id)
{
}
static inline void nvme_mpath_remove_disk(struct nvme_ns_head *head)
{
}
static inline void nvme_mpath_clear_current_path(struct nvme_ns *ns)
{
}
static inline void nvme_mpath_check_last_path(struct nvme_ns *ns)
{
}
static inline int nvme_mpath_init(struct nvme_ctrl *ctrl,
		struct nvme_id_ctrl *id)
{
	if (ctrl->subsys->cmic & (1 << 3))
		dev_warn(ctrl->device,
"Please enable CONFIG_NVME_MULTIPATH for full support of multi-port devices.\n");
	return 0;
}
static inline void nvme_mpath_uninit(struct nvme_ctrl *ctrl)
{
}
static inline void nvme_mpath_stop(struct nvme_ctrl *ctrl)
{
}
#endif /* CONFIG_NVME_MULTIPATH */

#ifdef CONFIG_NVM
int nvme_nvm_register(struct nvme_ns *ns, char *disk_name, int node);
void nvme_nvm_unregister(struct nvme_ns *ns);
extern const struct attribute_group nvme_nvm_attr_group;
int nvme_nvm_ioctl(struct nvme_ns *ns, unsigned int cmd, unsigned long arg);
#else
static inline int nvme_nvm_register(struct nvme_ns *ns, char *disk_name,
				    int node)
{
	return 0;
}

static inline void nvme_nvm_unregister(struct nvme_ns *ns) {};
static inline int nvme_nvm_ioctl(struct nvme_ns *ns, unsigned int cmd,
							unsigned long arg)
{
	return -ENOTTY;
}
#endif /* CONFIG_NVM */

static inline struct nvme_ns *nvme_get_ns_from_dev(struct device *dev)
{
	return dev_to_disk(dev)->private_data;
}

int __init nvme_core_init(void);
void __exit nvme_core_exit(void);

#endif /* _NVME_H */
Commit	Line	Data
bc50ad75	1	/* SPDX-License-Identifier: GPL-2.0 */
f11bb3e2 CH	2	/*
f11bb3e2 CH	3	* Copyright (c) 2011-2014, Intel Corporation.
f11bb3e2 CH	4	*/
	5
	6	#ifndef _NVME_H
	7	#define _NVME_H
	8
	9	#include <linux/nvme.h>
a6a5149b	10	#include <linux/cdev.h>
f11bb3e2 CH	11	#include <linux/pci.h>
	12	#include <linux/kref.h>
	13	#include <linux/blk-mq.h>
b0b4e09c	14	#include <linux/lightnvm.h>
a98e58e5	15	#include <linux/sed-opal.h>
b9e03857	16	#include <linux/fault-inject.h>
978628ec	17	#include <linux/rcupdate.h>
f11bb3e2	18
8ae4e447	19	extern unsigned int nvme_io_timeout;
f11bb3e2 CH	20	#define NVME_IO_TIMEOUT (nvme_io_timeout * HZ)
f11bb3e2 CH	21
8ae4e447	22	extern unsigned int admin_timeout;
21d34711 CH	23	#define ADMIN_TIMEOUT (admin_timeout * HZ)
21d34711 CH	24
038bd4cb SG	25	#define NVME_DEFAULT_KATO 5
	26	#define NVME_KATO_GRACE 10
	27
9a6327d2	28	extern struct workqueue_struct *nvme_wq;
b227c59b RS	29	extern struct workqueue_struct *nvme_reset_wq;
b227c59b RS	30	extern struct workqueue_struct *nvme_delete_wq;
9a6327d2	31
ca064085 MB	32	enum {
	33	NVME_NS_LBA = 0,
	34	NVME_NS_LIGHTNVM = 1,
	35	};
	36
f11bb3e2	37	/*
106198ed CH	38	* List of workarounds for devices that required behavior not specified in
106198ed CH	39	* the standard.
f11bb3e2	40	*/
106198ed CH	41	enum nvme_quirks {
	42	/*
	43	* Prefers I/O aligned to a stripe size specified in a vendor
	44	* specific Identify field.
	45	*/
	46	NVME_QUIRK_STRIPE_SIZE = (1 << 0),
540c801c KB	47
	48	/*
	49	* The controller doesn't handle Identify value others than 0 or 1
	50	* correctly.
	51	*/
	52	NVME_QUIRK_IDENTIFY_CNS = (1 << 1),
08095e70 KB	53
08095e70 KB	54	/*
e850fd16 CH	55	* The controller deterministically returns O's on reads to
e850fd16 CH	56	* logical blocks that deallocate was called on.
08095e70	57	*/
e850fd16	58	NVME_QUIRK_DEALLOCATE_ZEROES = (1 << 2),
54adc010 GP	59
	60	/*
	61	* The controller needs a delay before starts checking the device
	62	* readiness, which is done by reading the NVME_CSTS_RDY bit.
	63	*/
	64	NVME_QUIRK_DELAY_BEFORE_CHK_RDY = (1 << 3),
c5552fde AL	65
	66	/*
	67	* APST should not be used.
	68	*/
	69	NVME_QUIRK_NO_APST = (1 << 4),
ff5350a8 AL	70
	71	/*
	72	* The deepest sleep state should not be used.
	73	*/
	74	NVME_QUIRK_NO_DEEPEST_PS = (1 << 5),
608cc4b1 CH	75
	76	/*
	77	* Supports the LighNVM command set if indicated in vs[1].
	78	*/
	79	NVME_QUIRK_LIGHTNVM = (1 << 6),
9abd68ef JA	80
	81	/*
	82	* Set MEDIUM priority on SQ creation
	83	*/
	84	NVME_QUIRK_MEDIUM_PRIO_SQ = (1 << 7),
6299358d JD	85
	86	/*
	87	* Ignore device provided subnqn.
	88	*/
	89	NVME_QUIRK_IGNORE_DEV_SUBNQN = (1 << 8),
7b210e4e CH	90
	91	/*
	92	* Broken Write Zeroes.
	93	*/
	94	NVME_QUIRK_DISABLE_WRITE_ZEROES = (1 << 9),
106198ed CH	95	};
106198ed CH	96
d49187e9 CH	97	/*
	98	* Common request structure for NVMe passthrough. All drivers must have
	99	* this structure as the first member of their request-private data.
	100	*/
	101	struct nvme_request {
	102	struct nvme_command *cmd;
	103	union nvme_result result;
44e44b29	104	u8 retries;
27fa9bc5 CH	105	u8 flags;
27fa9bc5 CH	106	u16 status;
59e29ce6	107	struct nvme_ctrl *ctrl;
27fa9bc5 CH	108	};
27fa9bc5 CH	109
32acab31 CH	110	/*
	111	* Mark a bio as coming in through the mpath node.
	112	*/
	113	#define REQ_NVME_MPATH REQ_DRV
	114
27fa9bc5 CH	115	enum {
27fa9bc5 CH	116	NVME_REQ_CANCELLED = (1 << 0),
bb06ec31	117	NVME_REQ_USERCMD = (1 << 1),
d49187e9 CH	118	};
	119
	120	static inline struct nvme_request nvme_req(struct request req)
	121	{
	122	return blk_mq_rq_to_pdu(req);
	123	}
	124
5d87eb94 KB	125	static inline u16 nvme_req_qid(struct request *req)
	126	{
	127	if (!req->rq_disk)
	128	return 0;
	129	return blk_mq_unique_tag_to_hwq(blk_mq_unique_tag(req)) + 1;
	130	}
	131
54adc010 GP	132	/* The below value is the specific amount of delay needed before checking
	133	* readiness in case of the PCI_DEVICE(0x1c58, 0x0003), which needs the
	134	* NVME_QUIRK_DELAY_BEFORE_CHK_RDY quirk enabled. The value (in ms) was
	135	* found empirically.
	136	*/
8c97eecc	137	#define NVME_QUIRK_DELAY_AMOUNT 2300
54adc010	138
bb8d261e CH	139	enum nvme_ctrl_state {
	140	NVME_CTRL_NEW,
	141	NVME_CTRL_LIVE,
2b1b7e78	142	NVME_CTRL_ADMIN_ONLY, /* Only admin queue live */
bb8d261e	143	NVME_CTRL_RESETTING,
ad6a0a52	144	NVME_CTRL_CONNECTING,
bb8d261e	145	NVME_CTRL_DELETING,
0ff9d4e1	146	NVME_CTRL_DEAD,
bb8d261e CH	147	};
bb8d261e CH	148
1c63dc66	149	struct nvme_ctrl {
6e3ca03e	150	bool comp_seen;
bb8d261e	151	enum nvme_ctrl_state state;
bd4da3ab	152	bool identified;
bb8d261e	153	spinlock_t lock;
e7ad43c3	154	struct mutex scan_lock;
1c63dc66	155	const struct nvme_ctrl_ops *ops;
f11bb3e2	156	struct request_queue *admin_q;
07bfcd09	157	struct request_queue *connect_q;
f11bb3e2	158	struct device *dev;
f11bb3e2	159	int instance;
103e515e	160	int numa_node;
5bae7f73	161	struct blk_mq_tag_set *tagset;
34b6c231	162	struct blk_mq_tag_set *admin_tagset;
f11bb3e2	163	struct list_head namespaces;
765cc031	164	struct rw_semaphore namespaces_rwsem;
d22524a4	165	struct device ctrl_device;
5bae7f73	166	struct device device; / char device */
a6a5149b	167	struct cdev cdev;
d86c4d8e	168	struct work_struct reset_work;
c5017e85	169	struct work_struct delete_work;
1c63dc66	170
ab9e00cc CH	171	struct nvme_subsystem *subsys;
	172	struct list_head subsys_entry;
	173
4f1244c8	174	struct opal_dev *opal_dev;
a98e58e5	175
f11bb3e2	176	char name[12];
76e3914a	177	u16 cntlid;
5fd4ce1b CH	178
5fd4ce1b CH	179	u32 ctrl_config;
b6dccf7f	180	u16 mtfa;
d858e5f0	181	u32 queue_count;
5fd4ce1b	182
20d0dfe6	183	u64 cap;
5fd4ce1b	184	u32 page_size;
f11bb3e2	185	u32 max_hw_sectors;
943e942e	186	u32 max_segments;
49cd84b6	187	u16 crdt[3];
f11bb3e2	188	u16 oncs;
8a9ae523	189	u16 oacs;
f5d11840 JA	190	u16 nssa;
f5d11840 JA	191	u16 nr_streams;
0d0b660f	192	u32 max_namespaces;
6bf25d16	193	atomic_t abort_limit;
f11bb3e2	194	u8 vwc;
f3ca80fc	195	u32 vs;
07bfcd09	196	u32 sgls;
038bd4cb	197	u16 kas;
c5552fde AL	198	u8 npss;
c5552fde AL	199	u8 apsta;
c0561f82	200	u32 oaes;
e3d7874d	201	u32 aen_result;
3e53ba38	202	u32 ctratt;
07fbd32a	203	unsigned int shutdown_timeout;
038bd4cb	204	unsigned int kato;
f3ca80fc	205	bool subsystem;
106198ed	206	unsigned long quirks;
c5552fde	207	struct nvme_id_power_state psd[32];
84fef62d	208	struct nvme_effects_log *effects;
5955be21	209	struct work_struct scan_work;
f866fc42	210	struct work_struct async_event_work;
038bd4cb	211	struct delayed_work ka_work;
0a34e466	212	struct nvme_command ka_cmd;
b6dccf7f	213	struct work_struct fw_act_work;
30d90964	214	unsigned long events;
07bfcd09	215
0d0b660f CH	216	#ifdef CONFIG_NVME_MULTIPATH
	217	/* asymmetric namespace access: */
	218	u8 anacap;
	219	u8 anatt;
	220	u32 anagrpmax;
	221	u32 nanagrpid;
	222	struct mutex ana_lock;
	223	struct nvme_ana_rsp_hdr *ana_log_buf;
	224	size_t ana_log_size;
	225	struct timer_list anatt_timer;
	226	struct work_struct ana_work;
	227	#endif
	228
c5552fde AL	229	/* Power saving configuration */
c5552fde AL	230	u64 ps_max_latency_us;
76a5af84	231	bool apst_enabled;
c5552fde	232
044a9df1	233	/* PCIe only: */
fe6d53c9 CH	234	u32 hmpre;
fe6d53c9 CH	235	u32 hmmin;
044a9df1 CH	236	u32 hmminds;
044a9df1 CH	237	u16 hmmaxd;
fe6d53c9	238
07bfcd09 CH	239	/* Fabrics only */
	240	u16 sqsize;
	241	u32 ioccsz;
	242	u32 iorcsz;
	243	u16 icdoff;
	244	u16 maxcmd;
fdf9dfa8	245	int nr_reconnects;
07bfcd09	246	struct nvmf_ctrl_options *opts;
cb5b7262 JA	247
	248	struct page *discard_page;
	249	unsigned long discard_page_busy;
f11bb3e2 CH	250	};
f11bb3e2 CH	251
75c10e73 HR	252	enum nvme_iopolicy {
	253	NVME_IOPOLICY_NUMA,
	254	NVME_IOPOLICY_RR,
	255	};
	256
ab9e00cc CH	257	struct nvme_subsystem {
	258	int instance;
	259	struct device dev;
	260	/*
	261	* Because we unregister the device on the last put we need
	262	* a separate refcount.
	263	*/
	264	struct kref ref;
	265	struct list_head entry;
	266	struct mutex lock;
	267	struct list_head ctrls;
ed754e5d	268	struct list_head nsheads;
ab9e00cc CH	269	char subnqn[NVMF_NQN_SIZE];
	270	char serial[20];
	271	char model[40];
	272	char firmware_rev[8];
	273	u8 cmic;
	274	u16 vendor_id;
ed754e5d	275	struct ida ns_ida;
75c10e73 HR	276	#ifdef CONFIG_NVME_MULTIPATH
	277	enum nvme_iopolicy iopolicy;
	278	#endif
ab9e00cc CH	279	};
ab9e00cc CH	280
002fab04 CH	281	/*
	282	* Container structure for uniqueue namespace identifiers.
	283	*/
	284	struct nvme_ns_ids {
	285	u8 eui64[8];
	286	u8 nguid[16];
	287	uuid_t uuid;
	288	};
	289
ed754e5d CH	290	/*
	291	* Anchor structure for namespaces. There is one for each namespace in a
	292	* NVMe subsystem that any of our controllers can see, and the namespace
	293	* structure for each controller is chained of it. For private namespaces
	294	* there is a 1:1 relation to our namespace structures, that is ->list
	295	* only ever has a single entry for private namespaces.
	296	*/
	297	struct nvme_ns_head {
	298	struct list_head list;
	299	struct srcu_struct srcu;
	300	struct nvme_subsystem *subsys;
	301	unsigned ns_id;
	302	struct nvme_ns_ids ids;
	303	struct list_head entry;
	304	struct kref ref;
	305	int instance;
f3334447 CH	306	#ifdef CONFIG_NVME_MULTIPATH
	307	struct gendisk *disk;
	308	struct bio_list requeue_list;
	309	spinlock_t requeue_lock;
	310	struct work_struct requeue_work;
	311	struct mutex lock;
	312	struct nvme_ns __rcu *current_path[];
	313	#endif
ed754e5d CH	314	};
ed754e5d CH	315
b9e03857 TT	316	#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
	317	struct nvme_fault_inject {
	318	struct fault_attr attr;
	319	struct dentry *parent;
	320	bool dont_retry; /* DNR, do not retry */
	321	u16 status; /* status code */
	322	};
	323	#endif
	324
f11bb3e2 CH	325	struct nvme_ns {
	326	struct list_head list;
	327
1c63dc66	328	struct nvme_ctrl *ctrl;
f11bb3e2 CH	329	struct request_queue *queue;
f11bb3e2 CH	330	struct gendisk *disk;
0d0b660f CH	331	#ifdef CONFIG_NVME_MULTIPATH
	332	enum nvme_ana_state ana_state;
	333	u32 ana_grpid;
	334	#endif
ed754e5d	335	struct list_head siblings;
b0b4e09c	336	struct nvm_dev *ndev;
f11bb3e2	337	struct kref kref;
ed754e5d	338	struct nvme_ns_head *head;
f11bb3e2	339
f11bb3e2 CH	340	int lba_shift;
f11bb3e2 CH	341	u16 ms;
f5d11840 JA	342	u16 sgs;
f5d11840 JA	343	u32 sws;
f11bb3e2 CH	344	bool ext;
f11bb3e2 CH	345	u8 pi_type;
646017a6	346	unsigned long flags;
0d0b660f CH	347	#define NVME_NS_REMOVING 0
	348	#define NVME_NS_DEAD 1
	349	#define NVME_NS_ANA_PENDING 2
57eeaf8e	350	u16 noiob;
b9e03857 TT	351
	352	#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
	353	struct nvme_fault_inject fault_inject;
	354	#endif
	355
f11bb3e2 CH	356	};
f11bb3e2 CH	357
1c63dc66	358	struct nvme_ctrl_ops {
1a353d85	359	const char *name;
e439bb12	360	struct module *module;
d3d5b87d CH	361	unsigned int flags;
d3d5b87d CH	362	#define NVME_F_FABRICS (1 << 0)
c81bfba9	363	#define NVME_F_METADATA_SUPPORTED (1 << 1)
e0596ab2	364	#define NVME_F_PCI_P2PDMA (1 << 2)
1c63dc66	365	int (reg_read32)(struct nvme_ctrl ctrl, u32 off, u32 *val);
5fd4ce1b	366	int (reg_write32)(struct nvme_ctrl ctrl, u32 off, u32 val);
7fd8930f	367	int (reg_read64)(struct nvme_ctrl ctrl, u32 off, u64 *val);
1673f1f0	368	void (free_ctrl)(struct nvme_ctrl ctrl);
ad22c355	369	void (submit_async_event)(struct nvme_ctrl ctrl);
c5017e85	370	void (delete_ctrl)(struct nvme_ctrl ctrl);
1a353d85	371	int (get_address)(struct nvme_ctrl ctrl, char *buf, int size);
f11bb3e2 CH	372	};
f11bb3e2 CH	373
b9e03857 TT	374	#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
	375	void nvme_fault_inject_init(struct nvme_ns *ns);
	376	void nvme_fault_inject_fini(struct nvme_ns *ns);
	377	void nvme_should_fail(struct request *req);
	378	#else
	379	static inline void nvme_fault_inject_init(struct nvme_ns *ns) {}
	380	static inline void nvme_fault_inject_fini(struct nvme_ns *ns) {}
	381	static inline void nvme_should_fail(struct request *req) {}
	382	#endif
	383
f3ca80fc CH	384	static inline int nvme_reset_subsystem(struct nvme_ctrl *ctrl)
	385	{
	386	if (!ctrl->subsystem)
	387	return -ENOTTY;
	388	return ctrl->ops->reg_write32(ctrl, NVME_REG_NSSR, 0x4E564D65);
	389	}
	390
f11bb3e2 CH	391	static inline u64 nvme_block_nr(struct nvme_ns *ns, sector_t sector)
	392	{
	393	return (sector >> (ns->lba_shift - 9));
	394	}
	395
27fa9bc5 CH	396	static inline void nvme_end_request(struct request *req, __le16 status,
27fa9bc5 CH	397	union nvme_result result)
15a190f7	398	{
27fa9bc5	399	struct nvme_request *rq = nvme_req(req);
15a190f7	400
27fa9bc5 CH	401	rq->status = le16_to_cpu(status) >> 1;
27fa9bc5 CH	402	rq->result = result;
b9e03857 TT	403	/* inject error when permitted by fault injection framework */
b9e03857 TT	404	nvme_should_fail(req);
08e0029a	405	blk_mq_complete_request(req);
7688faa6 CH	406	}
7688faa6 CH	407
d22524a4 CH	408	static inline void nvme_get_ctrl(struct nvme_ctrl *ctrl)
	409	{
	410	get_device(ctrl->device);
	411	}
	412
	413	static inline void nvme_put_ctrl(struct nvme_ctrl *ctrl)
	414	{
	415	put_device(ctrl->device);
	416	}
	417
77f02a7a	418	void nvme_complete_rq(struct request *req);
7baa8572	419	bool nvme_cancel_request(struct request req, void data, bool reserved);
bb8d261e CH	420	bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
bb8d261e CH	421	enum nvme_ctrl_state new_state);
5fd4ce1b CH	422	int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap);
	423	int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap);
	424	int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl);
f3ca80fc CH	425	int nvme_init_ctrl(struct nvme_ctrl ctrl, struct device dev,
f3ca80fc CH	426	const struct nvme_ctrl_ops *ops, unsigned long quirks);
53029b04	427	void nvme_uninit_ctrl(struct nvme_ctrl *ctrl);
d09f2b45 SG	428	void nvme_start_ctrl(struct nvme_ctrl *ctrl);
d09f2b45 SG	429	void nvme_stop_ctrl(struct nvme_ctrl *ctrl);
1673f1f0	430	void nvme_put_ctrl(struct nvme_ctrl *ctrl);
7fd8930f	431	int nvme_init_identify(struct nvme_ctrl *ctrl);
5bae7f73	432
5bae7f73	433	void nvme_remove_namespaces(struct nvme_ctrl *ctrl);
1673f1f0	434
4f1244c8 CH	435	int nvme_sec_submit(void data, u16 spsp, u8 secp, void buffer, size_t len,
4f1244c8 CH	436	bool send);
a98e58e5	437
7bf58533	438	void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
287a63eb	439	volatile union nvme_result *res);
f866fc42	440
25646264 KB	441	void nvme_stop_queues(struct nvme_ctrl *ctrl);
25646264 KB	442	void nvme_start_queues(struct nvme_ctrl *ctrl);
69d9a99c	443	void nvme_kill_queues(struct nvme_ctrl *ctrl);
302ad8cc KB	444	void nvme_unfreeze(struct nvme_ctrl *ctrl);
	445	void nvme_wait_freeze(struct nvme_ctrl *ctrl);
	446	void nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout);
	447	void nvme_start_freeze(struct nvme_ctrl *ctrl);
363c9aac	448
eb71f435	449	#define NVME_QID_ANY -1
4160982e	450	struct request nvme_alloc_request(struct request_queue q,
9a95e4ef	451	struct nvme_command *cmd, blk_mq_req_flags_t flags, int qid);
f7f1fc36	452	void nvme_cleanup_cmd(struct request *req);
fc17b653	453	blk_status_t nvme_setup_cmd(struct nvme_ns ns, struct request req,
8093f7ca	454	struct nvme_command *cmd);
f11bb3e2 CH	455	int nvme_submit_sync_cmd(struct request_queue q, struct nvme_command cmd,
	456	void *buf, unsigned bufflen);
	457	int __nvme_submit_sync_cmd(struct request_queue q, struct nvme_command cmd,
d49187e9	458	union nvme_result result, void buffer, unsigned bufflen,
9a95e4ef	459	unsigned timeout, int qid, int at_head,
6287b51c	460	blk_mq_req_flags_t flags, bool poll);
9a0be7ab	461	int nvme_set_queue_count(struct nvme_ctrl ctrl, int count);
038bd4cb	462	void nvme_stop_keep_alive(struct nvme_ctrl *ctrl);
d86c4d8e	463	int nvme_reset_ctrl(struct nvme_ctrl *ctrl);
79c48ccf	464	int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl);
c5017e85	465	int nvme_delete_ctrl(struct nvme_ctrl *ctrl);
f11bb3e2	466
0e98719b CH	467	int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp,
0e98719b CH	468	void *log, size_t size, u64 offset);
d558fb51	469
33b14f67	470	extern const struct attribute_group *nvme_ns_id_attr_groups[];
32acab31 CH	471	extern const struct block_device_operations nvme_ns_head_ops;
	472
	473	#ifdef CONFIG_NVME_MULTIPATH
0d0b660f	474	bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl);
a785dbcc KB	475	void nvme_set_disk_name(char disk_name, struct nvme_ns ns,
a785dbcc KB	476	struct nvme_ctrl ctrl, int flags);
32acab31	477	void nvme_failover_req(struct request *req);
32acab31 CH	478	void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl);
32acab31 CH	479	int nvme_mpath_alloc_disk(struct nvme_ctrl ctrl,struct nvme_ns_head head);
0d0b660f	480	void nvme_mpath_add_disk(struct nvme_ns ns, struct nvme_id_ns id);
32acab31	481	void nvme_mpath_remove_disk(struct nvme_ns_head *head);
0d0b660f CH	482	int nvme_mpath_init(struct nvme_ctrl ctrl, struct nvme_id_ctrl id);
	483	void nvme_mpath_uninit(struct nvme_ctrl *ctrl);
	484	void nvme_mpath_stop(struct nvme_ctrl *ctrl);
f3334447	485	void nvme_mpath_clear_current_path(struct nvme_ns *ns);
32acab31	486	struct nvme_ns nvme_find_path(struct nvme_ns_head head);
479a322f SG	487
	488	static inline void nvme_mpath_check_last_path(struct nvme_ns *ns)
	489	{
	490	struct nvme_ns_head *head = ns->head;
	491
	492	if (head->disk && list_empty(&head->list))
	493	kblockd_schedule_work(&head->requeue_work);
	494	}
	495
0d0b660f CH	496	extern struct device_attribute dev_attr_ana_grpid;
0d0b660f CH	497	extern struct device_attribute dev_attr_ana_state;
75c10e73	498	extern struct device_attribute subsys_attr_iopolicy;
0d0b660f	499
32acab31	500	#else
0d0b660f CH	501	static inline bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl)
	502	{
	503	return false;
	504	}
a785dbcc KB	505	/*
	506	* Without the multipath code enabled, multiple controller per subsystems are
	507	* visible as devices and thus we cannot use the subsystem instance.
	508	*/
	509	static inline void nvme_set_disk_name(char disk_name, struct nvme_ns ns,
	510	struct nvme_ctrl ctrl, int flags)
	511	{
	512	sprintf(disk_name, "nvme%dn%d", ctrl->instance, ns->head->instance);
	513	}
	514
32acab31 CH	515	static inline void nvme_failover_req(struct request *req)
	516	{
	517	}
32acab31 CH	518	static inline void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)
	519	{
	520	}
	521	static inline int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl,
	522	struct nvme_ns_head *head)
	523	{
	524	return 0;
	525	}
0d0b660f CH	526	static inline void nvme_mpath_add_disk(struct nvme_ns *ns,
0d0b660f CH	527	struct nvme_id_ns *id)
32acab31 CH	528	{
	529	}
	530	static inline void nvme_mpath_remove_disk(struct nvme_ns_head *head)
	531	{
	532	}
	533	static inline void nvme_mpath_clear_current_path(struct nvme_ns *ns)
479a322f SG	534	{
	535	}
	536	static inline void nvme_mpath_check_last_path(struct nvme_ns *ns)
32acab31 CH	537	{
32acab31 CH	538	}
0d0b660f CH	539	static inline int nvme_mpath_init(struct nvme_ctrl *ctrl,
	540	struct nvme_id_ctrl *id)
	541	{
14a1336e CH	542	if (ctrl->subsys->cmic & (1 << 3))
	543	dev_warn(ctrl->device,
	544	"Please enable CONFIG_NVME_MULTIPATH for full support of multi-port devices.\n");
0d0b660f CH	545	return 0;
	546	}
	547	static inline void nvme_mpath_uninit(struct nvme_ctrl *ctrl)
	548	{
	549	}
	550	static inline void nvme_mpath_stop(struct nvme_ctrl *ctrl)
	551	{
	552	}
32acab31 CH	553	#endif /* CONFIG_NVME_MULTIPATH */
32acab31 CH	554
c4699e70	555	#ifdef CONFIG_NVM
3dc87dd0	556	int nvme_nvm_register(struct nvme_ns ns, char disk_name, int node);
b0b4e09c	557	void nvme_nvm_unregister(struct nvme_ns *ns);
33b14f67	558	extern const struct attribute_group nvme_nvm_attr_group;
84d4add7	559	int nvme_nvm_ioctl(struct nvme_ns *ns, unsigned int cmd, unsigned long arg);
c4699e70	560	#else
b0b4e09c	561	static inline int nvme_nvm_register(struct nvme_ns ns, char disk_name,
3dc87dd0	562	int node)
c4699e70 KB	563	{
	564	return 0;
	565	}
	566
b0b4e09c	567	static inline void nvme_nvm_unregister(struct nvme_ns *ns) {};
84d4add7 MB	568	static inline int nvme_nvm_ioctl(struct nvme_ns *ns, unsigned int cmd,
	569	unsigned long arg)
	570	{
	571	return -ENOTTY;
	572	}
3dc87dd0 MB	573	#endif /* CONFIG_NVM */
3dc87dd0 MB	574
40267efd SL	575	static inline struct nvme_ns nvme_get_ns_from_dev(struct device dev)
	576	{
	577	return dev_to_disk(dev)->private_data;
	578	}
ca064085	579
5bae7f73	580	int __init nvme_core_init(void);
8eb5d89f	581	void __exit nvme_core_exit(void);
5bae7f73	582
f11bb3e2	583	#endif /* _NVME_H */