[mirror_qemu.git] / hw / nvme / nvme.h

/*
 * QEMU NVM Express
 *
 * Copyright (c) 2012 Intel Corporation
 * Copyright (c) 2021 Minwoo Im
 * Copyright (c) 2021 Samsung Electronics Co., Ltd.
 *
 * Authors:
 *   Keith Busch            <kbusch@kernel.org>
 *   Klaus Jensen           <k.jensen@samsung.com>
 *   Gollu Appalanaidu      <anaidu.gollu@samsung.com>
 *   Dmitry Fomichev        <dmitry.fomichev@wdc.com>
 *   Minwoo Im              <minwoo.im.dev@gmail.com>
 *
 * This code is licensed under the GNU GPL v2 or later.
 */

#ifndef HW_NVME_NVME_H
#define HW_NVME_NVME_H

#include "qemu/uuid.h"
#include "hw/pci/pci.h"
#include "hw/block/block.h"

#include "block/nvme.h"

#define NVME_MAX_CONTROLLERS 256
#define NVME_MAX_NAMESPACES  256
#define NVME_EUI64_DEFAULT ((uint64_t)0x5254000000000000)

QEMU_BUILD_BUG_ON(NVME_MAX_NAMESPACES > NVME_NSID_BROADCAST - 1);

typedef struct NvmeCtrl NvmeCtrl;
typedef struct NvmeNamespace NvmeNamespace;

#define TYPE_NVME_BUS "nvme-bus"
OBJECT_DECLARE_SIMPLE_TYPE(NvmeBus, NVME_BUS)

typedef struct NvmeBus {
    BusState parent_bus;
} NvmeBus;

#define TYPE_NVME_SUBSYS "nvme-subsys"
#define NVME_SUBSYS(obj) \
    OBJECT_CHECK(NvmeSubsystem, (obj), TYPE_NVME_SUBSYS)
#define SUBSYS_SLOT_RSVD (void *)0xFFFF

typedef struct NvmeSubsystem {
    DeviceState parent_obj;
    NvmeBus     bus;
    uint8_t     subnqn[256];
    char        *serial;

    NvmeCtrl      *ctrls[NVME_MAX_CONTROLLERS];
    NvmeNamespace *namespaces[NVME_MAX_NAMESPACES + 1];

    struct {
        char *nqn;
    } params;
} NvmeSubsystem;

int nvme_subsys_register_ctrl(NvmeCtrl *n, Error **errp);
void nvme_subsys_unregister_ctrl(NvmeSubsystem *subsys, NvmeCtrl *n);

static inline NvmeCtrl *nvme_subsys_ctrl(NvmeSubsystem *subsys,
                                         uint32_t cntlid)
{
    if (!subsys || cntlid >= NVME_MAX_CONTROLLERS) {
        return NULL;
    }

    if (subsys->ctrls[cntlid] == SUBSYS_SLOT_RSVD) {
        return NULL;
    }

    return subsys->ctrls[cntlid];
}

static inline NvmeNamespace *nvme_subsys_ns(NvmeSubsystem *subsys,
                                            uint32_t nsid)
{
    if (!subsys || !nsid || nsid > NVME_MAX_NAMESPACES) {
        return NULL;
    }

    return subsys->namespaces[nsid];
}

#define TYPE_NVME_NS "nvme-ns"
#define NVME_NS(obj) \
    OBJECT_CHECK(NvmeNamespace, (obj), TYPE_NVME_NS)

typedef struct NvmeZone {
    NvmeZoneDescr   d;
    uint64_t        w_ptr;
    QTAILQ_ENTRY(NvmeZone) entry;
} NvmeZone;

typedef struct NvmeNamespaceParams {
    bool     detached;
    bool     shared;
    uint32_t nsid;
    QemuUUID uuid;
    uint64_t eui64;
    bool     eui64_default;

    uint16_t ms;
    uint8_t  mset;
    uint8_t  pi;
    uint8_t  pil;
    uint8_t  pif;

    uint16_t mssrl;
    uint32_t mcl;
    uint8_t  msrc;

    bool     zoned;
    bool     cross_zone_read;
    uint64_t zone_size_bs;
    uint64_t zone_cap_bs;
    uint32_t max_active_zones;
    uint32_t max_open_zones;
    uint32_t zd_extension_size;

    uint32_t numzrwa;
    uint64_t zrwas;
    uint64_t zrwafg;
} NvmeNamespaceParams;

typedef struct NvmeNamespace {
    DeviceState  parent_obj;
    BlockConf    blkconf;
    int32_t      bootindex;
    int64_t      size;
    int64_t      moff;
    NvmeIdNs     id_ns;
    NvmeIdNsNvm  id_ns_nvm;
    NvmeLBAF     lbaf;
    unsigned int nlbaf;
    size_t       lbasz;
    const uint32_t *iocs;
    uint8_t      csi;
    uint16_t     status;
    int          attached;
    uint8_t      pif;

    struct {
        uint16_t zrwas;
        uint16_t zrwafg;
        uint32_t numzrwa;
    } zns;

    QTAILQ_ENTRY(NvmeNamespace) entry;

    NvmeIdNsZoned   *id_ns_zoned;
    NvmeZone        *zone_array;
    QTAILQ_HEAD(, NvmeZone) exp_open_zones;
    QTAILQ_HEAD(, NvmeZone) imp_open_zones;
    QTAILQ_HEAD(, NvmeZone) closed_zones;
    QTAILQ_HEAD(, NvmeZone) full_zones;
    uint32_t        num_zones;
    uint64_t        zone_size;
    uint64_t        zone_capacity;
    uint32_t        zone_size_log2;
    uint8_t         *zd_extensions;
    int32_t         nr_open_zones;
    int32_t         nr_active_zones;

    NvmeNamespaceParams params;

    struct {
        uint32_t err_rec;
    } features;
} NvmeNamespace;

static inline uint32_t nvme_nsid(NvmeNamespace *ns)
{
    if (ns) {
        return ns->params.nsid;
    }

    return 0;
}

static inline size_t nvme_l2b(NvmeNamespace *ns, uint64_t lba)
{
    return lba << ns->lbaf.ds;
}

static inline size_t nvme_m2b(NvmeNamespace *ns, uint64_t lba)
{
    return ns->lbaf.ms * lba;
}

static inline int64_t nvme_moff(NvmeNamespace *ns, uint64_t lba)
{
    return ns->moff + nvme_m2b(ns, lba);
}

static inline bool nvme_ns_ext(NvmeNamespace *ns)
{
    return !!NVME_ID_NS_FLBAS_EXTENDED(ns->id_ns.flbas);
}

static inline NvmeZoneState nvme_get_zone_state(NvmeZone *zone)
{
    return zone->d.zs >> 4;
}

static inline void nvme_set_zone_state(NvmeZone *zone, NvmeZoneState state)
{
    zone->d.zs = state << 4;
}

static inline uint64_t nvme_zone_rd_boundary(NvmeNamespace *ns, NvmeZone *zone)
{
    return zone->d.zslba + ns->zone_size;
}

static inline uint64_t nvme_zone_wr_boundary(NvmeZone *zone)
{
    return zone->d.zslba + zone->d.zcap;
}

static inline bool nvme_wp_is_valid(NvmeZone *zone)
{
    uint8_t st = nvme_get_zone_state(zone);

    return st != NVME_ZONE_STATE_FULL &&
           st != NVME_ZONE_STATE_READ_ONLY &&
           st != NVME_ZONE_STATE_OFFLINE;
}

static inline uint8_t *nvme_get_zd_extension(NvmeNamespace *ns,
                                             uint32_t zone_idx)
{
    return &ns->zd_extensions[zone_idx * ns->params.zd_extension_size];
}

static inline void nvme_aor_inc_open(NvmeNamespace *ns)
{
    assert(ns->nr_open_zones >= 0);
    if (ns->params.max_open_zones) {
        ns->nr_open_zones++;
        assert(ns->nr_open_zones <= ns->params.max_open_zones);
    }
}

static inline void nvme_aor_dec_open(NvmeNamespace *ns)
{
    if (ns->params.max_open_zones) {
        assert(ns->nr_open_zones > 0);
        ns->nr_open_zones--;
    }
    assert(ns->nr_open_zones >= 0);
}

static inline void nvme_aor_inc_active(NvmeNamespace *ns)
{
    assert(ns->nr_active_zones >= 0);
    if (ns->params.max_active_zones) {
        ns->nr_active_zones++;
        assert(ns->nr_active_zones <= ns->params.max_active_zones);
    }
}

static inline void nvme_aor_dec_active(NvmeNamespace *ns)
{
    if (ns->params.max_active_zones) {
        assert(ns->nr_active_zones > 0);
        ns->nr_active_zones--;
        assert(ns->nr_active_zones >= ns->nr_open_zones);
    }
    assert(ns->nr_active_zones >= 0);
}

void nvme_ns_init_format(NvmeNamespace *ns);
int nvme_ns_setup(NvmeNamespace *ns, Error **errp);
void nvme_ns_drain(NvmeNamespace *ns);
void nvme_ns_shutdown(NvmeNamespace *ns);
void nvme_ns_cleanup(NvmeNamespace *ns);

typedef struct NvmeAsyncEvent {
    QTAILQ_ENTRY(NvmeAsyncEvent) entry;
    NvmeAerResult result;
} NvmeAsyncEvent;

enum {
    NVME_SG_ALLOC = 1 << 0,
    NVME_SG_DMA   = 1 << 1,
};

typedef struct NvmeSg {
    int flags;

    union {
        QEMUSGList   qsg;
        QEMUIOVector iov;
    };
} NvmeSg;

typedef enum NvmeTxDirection {
    NVME_TX_DIRECTION_TO_DEVICE   = 0,
    NVME_TX_DIRECTION_FROM_DEVICE = 1,
} NvmeTxDirection;

typedef struct NvmeRequest {
    struct NvmeSQueue       *sq;
    struct NvmeNamespace    *ns;
    BlockAIOCB              *aiocb;
    uint16_t                status;
    void                    *opaque;
    NvmeCqe                 cqe;
    NvmeCmd                 cmd;
    BlockAcctCookie         acct;
    NvmeSg                  sg;
    QTAILQ_ENTRY(NvmeRequest)entry;
} NvmeRequest;

typedef struct NvmeBounceContext {
    NvmeRequest *req;

    struct {
        QEMUIOVector iov;
        uint8_t *bounce;
    } data, mdata;
} NvmeBounceContext;

static inline const char *nvme_adm_opc_str(uint8_t opc)
{
    switch (opc) {
    case NVME_ADM_CMD_DELETE_SQ:        return "NVME_ADM_CMD_DELETE_SQ";
    case NVME_ADM_CMD_CREATE_SQ:        return "NVME_ADM_CMD_CREATE_SQ";
    case NVME_ADM_CMD_GET_LOG_PAGE:     return "NVME_ADM_CMD_GET_LOG_PAGE";
    case NVME_ADM_CMD_DELETE_CQ:        return "NVME_ADM_CMD_DELETE_CQ";
    case NVME_ADM_CMD_CREATE_CQ:        return "NVME_ADM_CMD_CREATE_CQ";
    case NVME_ADM_CMD_IDENTIFY:         return "NVME_ADM_CMD_IDENTIFY";
    case NVME_ADM_CMD_ABORT:            return "NVME_ADM_CMD_ABORT";
    case NVME_ADM_CMD_SET_FEATURES:     return "NVME_ADM_CMD_SET_FEATURES";
    case NVME_ADM_CMD_GET_FEATURES:     return "NVME_ADM_CMD_GET_FEATURES";
    case NVME_ADM_CMD_ASYNC_EV_REQ:     return "NVME_ADM_CMD_ASYNC_EV_REQ";
    case NVME_ADM_CMD_NS_ATTACHMENT:    return "NVME_ADM_CMD_NS_ATTACHMENT";
    case NVME_ADM_CMD_VIRT_MNGMT:       return "NVME_ADM_CMD_VIRT_MNGMT";
    case NVME_ADM_CMD_FORMAT_NVM:       return "NVME_ADM_CMD_FORMAT_NVM";
    default:                            return "NVME_ADM_CMD_UNKNOWN";
    }
}

static inline const char *nvme_io_opc_str(uint8_t opc)
{
    switch (opc) {
    case NVME_CMD_FLUSH:            return "NVME_NVM_CMD_FLUSH";
    case NVME_CMD_WRITE:            return "NVME_NVM_CMD_WRITE";
    case NVME_CMD_READ:             return "NVME_NVM_CMD_READ";
    case NVME_CMD_COMPARE:          return "NVME_NVM_CMD_COMPARE";
    case NVME_CMD_WRITE_ZEROES:     return "NVME_NVM_CMD_WRITE_ZEROES";
    case NVME_CMD_DSM:              return "NVME_NVM_CMD_DSM";
    case NVME_CMD_VERIFY:           return "NVME_NVM_CMD_VERIFY";
    case NVME_CMD_COPY:             return "NVME_NVM_CMD_COPY";
    case NVME_CMD_ZONE_MGMT_SEND:   return "NVME_ZONED_CMD_MGMT_SEND";
    case NVME_CMD_ZONE_MGMT_RECV:   return "NVME_ZONED_CMD_MGMT_RECV";
    case NVME_CMD_ZONE_APPEND:      return "NVME_ZONED_CMD_ZONE_APPEND";
    default:                        return "NVME_NVM_CMD_UNKNOWN";
    }
}

typedef struct NvmeSQueue {
    struct NvmeCtrl *ctrl;
    uint16_t    sqid;
    uint16_t    cqid;
    uint32_t    head;
    uint32_t    tail;
    uint32_t    size;
    uint64_t    dma_addr;
    QEMUTimer   *timer;
    NvmeRequest *io_req;
    QTAILQ_HEAD(, NvmeRequest) req_list;
    QTAILQ_HEAD(, NvmeRequest) out_req_list;
    QTAILQ_ENTRY(NvmeSQueue) entry;
} NvmeSQueue;

typedef struct NvmeCQueue {
    struct NvmeCtrl *ctrl;
    uint8_t     phase;
    uint16_t    cqid;
    uint16_t    irq_enabled;
    uint32_t    head;
    uint32_t    tail;
    uint32_t    vector;
    uint32_t    size;
    uint64_t    dma_addr;
    QEMUTimer   *timer;
    QTAILQ_HEAD(, NvmeSQueue) sq_list;
    QTAILQ_HEAD(, NvmeRequest) req_list;
} NvmeCQueue;

#define TYPE_NVME "nvme"
#define NVME(obj) \
        OBJECT_CHECK(NvmeCtrl, (obj), TYPE_NVME)

typedef struct NvmeParams {
    char     *serial;
    uint32_t num_queues; /* deprecated since 5.1 */
    uint32_t max_ioqpairs;
    uint16_t msix_qsize;
    uint32_t cmb_size_mb;
    uint8_t  aerl;
    uint32_t aer_max_queued;
    uint8_t  mdts;
    uint8_t  vsl;
    bool     use_intel_id;
    uint8_t  zasl;
    bool     auto_transition_zones;
    bool     legacy_cmb;
    uint8_t  sriov_max_vfs;
    uint16_t sriov_vq_flexible;
    uint16_t sriov_vi_flexible;
    uint8_t  sriov_max_vq_per_vf;
    uint8_t  sriov_max_vi_per_vf;
} NvmeParams;

typedef struct NvmeCtrl {
    PCIDevice    parent_obj;
    MemoryRegion bar0;
    MemoryRegion iomem;
    NvmeBar      bar;
    NvmeParams   params;
    NvmeBus      bus;

    uint16_t    cntlid;
    bool        qs_created;
    uint32_t    page_size;
    uint16_t    page_bits;
    uint16_t    max_prp_ents;
    uint16_t    cqe_size;
    uint16_t    sqe_size;
    uint32_t    max_q_ents;
    uint8_t     outstanding_aers;
    uint32_t    irq_status;
    int         cq_pending;
    uint64_t    host_timestamp;                 /* Timestamp sent by the host */
    uint64_t    timestamp_set_qemu_clock_ms;    /* QEMU clock time */
    uint64_t    starttime_ms;
    uint16_t    temperature;
    uint8_t     smart_critical_warning;
    uint32_t    conf_msix_qsize;
    uint32_t    conf_ioqpairs;

    struct {
        MemoryRegion mem;
        uint8_t      *buf;
        bool         cmse;
        hwaddr       cba;
    } cmb;

    struct {
        HostMemoryBackend *dev;
        bool              cmse;
        hwaddr            cba;
    } pmr;

    uint8_t     aer_mask;
    NvmeRequest **aer_reqs;
    QTAILQ_HEAD(, NvmeAsyncEvent) aer_queue;
    int         aer_queued;

    uint32_t    dmrsl;

    /* Namespace ID is started with 1 so bitmap should be 1-based */
#define NVME_CHANGED_NSID_SIZE  (NVME_MAX_NAMESPACES + 1)
    DECLARE_BITMAP(changed_nsids, NVME_CHANGED_NSID_SIZE);

    NvmeSubsystem   *subsys;

    NvmeNamespace   namespace;
    NvmeNamespace   *namespaces[NVME_MAX_NAMESPACES + 1];
    NvmeSQueue      **sq;
    NvmeCQueue      **cq;
    NvmeSQueue      admin_sq;
    NvmeCQueue      admin_cq;
    NvmeIdCtrl      id_ctrl;

    struct {
        struct {
            uint16_t temp_thresh_hi;
            uint16_t temp_thresh_low;
        };

        uint32_t                async_config;
        NvmeHostBehaviorSupport hbs;
    } features;

    NvmePriCtrlCap  pri_ctrl_cap;
    NvmeSecCtrlList sec_ctrl_list;
    struct {
        uint16_t    vqrfap;
        uint16_t    virfap;
    } next_pri_ctrl_cap;    /* These override pri_ctrl_cap after reset */
} NvmeCtrl;

typedef enum NvmeResetType {
    NVME_RESET_FUNCTION   = 0,
    NVME_RESET_CONTROLLER = 1,
} NvmeResetType;

static inline NvmeNamespace *nvme_ns(NvmeCtrl *n, uint32_t nsid)
{
    if (!nsid || nsid > NVME_MAX_NAMESPACES) {
        return NULL;
    }

    return n->namespaces[nsid];
}

static inline NvmeCQueue *nvme_cq(NvmeRequest *req)
{
    NvmeSQueue *sq = req->sq;
    NvmeCtrl *n = sq->ctrl;

    return n->cq[sq->cqid];
}

static inline NvmeCtrl *nvme_ctrl(NvmeRequest *req)
{
    NvmeSQueue *sq = req->sq;
    return sq->ctrl;
}

static inline uint16_t nvme_cid(NvmeRequest *req)
{
    if (!req) {
        return 0xffff;
    }

    return le16_to_cpu(req->cqe.cid);
}

static inline NvmeSecCtrlEntry *nvme_sctrl(NvmeCtrl *n)
{
    PCIDevice *pci_dev = &n->parent_obj;
    NvmeCtrl *pf = NVME(pcie_sriov_get_pf(pci_dev));

    if (pci_is_vf(pci_dev)) {
        return &pf->sec_ctrl_list.sec[pcie_sriov_vf_number(pci_dev)];
    }

    return NULL;
}

static inline NvmeSecCtrlEntry *nvme_sctrl_for_cntlid(NvmeCtrl *n,
                                                      uint16_t cntlid)
{
    NvmeSecCtrlList *list = &n->sec_ctrl_list;
    uint8_t i;

    for (i = 0; i < list->numcntl; i++) {
        if (le16_to_cpu(list->sec[i].scid) == cntlid) {
            return &list->sec[i];
        }
    }

    return NULL;
}

void nvme_attach_ns(NvmeCtrl *n, NvmeNamespace *ns);
uint16_t nvme_bounce_data(NvmeCtrl *n, void *ptr, uint32_t len,
                          NvmeTxDirection dir, NvmeRequest *req);
uint16_t nvme_bounce_mdata(NvmeCtrl *n, void *ptr, uint32_t len,
                           NvmeTxDirection dir, NvmeRequest *req);
void nvme_rw_complete_cb(void *opaque, int ret);
uint16_t nvme_map_dptr(NvmeCtrl *n, NvmeSg *sg, size_t len,
                       NvmeCmd *cmd);

#endif /* HW_NVME_NVME_H */
Commit	Line	Data
d88e784f KJ	1	/*
	2	* QEMU NVM Express
	3	*
	4	* Copyright (c) 2012 Intel Corporation
	5	* Copyright (c) 2021 Minwoo Im
	6	* Copyright (c) 2021 Samsung Electronics Co., Ltd.
	7	*
	8	* Authors:
	9	* Keith Busch <kbusch@kernel.org>
	10	* Klaus Jensen <k.jensen@samsung.com>
	11	* Gollu Appalanaidu <anaidu.gollu@samsung.com>
	12	* Dmitry Fomichev <dmitry.fomichev@wdc.com>
	13	* Minwoo Im <minwoo.im.dev@gmail.com>
	14	*
	15	* This code is licensed under the GNU GPL v2 or later.
	16	*/
	17
52581c71 MA	18	#ifndef HW_NVME_NVME_H
52581c71 MA	19	#define HW_NVME_NVME_H
1065abfb	20
d88e784f	21	#include "qemu/uuid.h"
146f720c	22	#include "hw/pci/pci.h"
d88e784f KJ	23	#include "hw/block/block.h"
	24
	25	#include "block/nvme.h"
7f0f1ace	26
44c2c094	27	#define NVME_MAX_CONTROLLERS 256
d88e784f	28	#define NVME_MAX_NAMESPACES 256
3276dde4	29	#define NVME_EUI64_DEFAULT ((uint64_t)0x5254000000000000)
a479335b	30
38f4ac65 KJ	31	QEMU_BUILD_BUG_ON(NVME_MAX_NAMESPACES > NVME_NSID_BROADCAST - 1);
38f4ac65 KJ	32
d88e784f KJ	33	typedef struct NvmeCtrl NvmeCtrl;
	34	typedef struct NvmeNamespace NvmeNamespace;
	35
5ffbaeed KJ	36	#define TYPE_NVME_BUS "nvme-bus"
	37	OBJECT_DECLARE_SIMPLE_TYPE(NvmeBus, NVME_BUS)
	38
	39	typedef struct NvmeBus {
	40	BusState parent_bus;
	41	} NvmeBus;
	42
d88e784f KJ	43	#define TYPE_NVME_SUBSYS "nvme-subsys"
	44	#define NVME_SUBSYS(obj) \
	45	OBJECT_CHECK(NvmeSubsystem, (obj), TYPE_NVME_SUBSYS)
99f48ae7	46	#define SUBSYS_SLOT_RSVD (void *)0xFFFF
d88e784f KJ	47
	48	typedef struct NvmeSubsystem {
	49	DeviceState parent_obj;
5ffbaeed	50	NvmeBus bus;
d88e784f	51	uint8_t subnqn[256];
a859eb9f	52	char *serial;
d88e784f KJ	53
	54	NvmeCtrl *ctrls[NVME_MAX_CONTROLLERS];
	55	NvmeNamespace *namespaces[NVME_MAX_NAMESPACES + 1];
	56
	57	struct {
	58	char *nqn;
	59	} params;
	60	} NvmeSubsystem;
	61
	62	int nvme_subsys_register_ctrl(NvmeCtrl n, Error *errp);
b0fde9e8	63	void nvme_subsys_unregister_ctrl(NvmeSubsystem subsys, NvmeCtrl n);
d88e784f KJ	64
	65	static inline NvmeCtrl nvme_subsys_ctrl(NvmeSubsystem subsys,
	66	uint32_t cntlid)
	67	{
	68	if (!subsys \|\| cntlid >= NVME_MAX_CONTROLLERS) {
	69	return NULL;
	70	}
	71
99f48ae7 LM	72	if (subsys->ctrls[cntlid] == SUBSYS_SLOT_RSVD) {
	73	return NULL;
	74	}
	75
d88e784f KJ	76	return subsys->ctrls[cntlid];
	77	}
	78
	79	static inline NvmeNamespace nvme_subsys_ns(NvmeSubsystem subsys,
	80	uint32_t nsid)
	81	{
	82	if (!subsys \|\| !nsid \|\| nsid > NVME_MAX_NAMESPACES) {
	83	return NULL;
	84	}
	85
	86	return subsys->namespaces[nsid];
	87	}
	88
	89	#define TYPE_NVME_NS "nvme-ns"
	90	#define NVME_NS(obj) \
	91	OBJECT_CHECK(NvmeNamespace, (obj), TYPE_NVME_NS)
	92
	93	typedef struct NvmeZone {
	94	NvmeZoneDescr d;
	95	uint64_t w_ptr;
	96	QTAILQ_ENTRY(NvmeZone) entry;
	97	} NvmeZone;
	98
	99	typedef struct NvmeNamespaceParams {
	100	bool detached;
	101	bool shared;
	102	uint32_t nsid;
	103	QemuUUID uuid;
6870cfb8	104	uint64_t eui64;
3276dde4	105	bool eui64_default;
d88e784f KJ	106
	107	uint16_t ms;
	108	uint8_t mset;
	109	uint8_t pi;
	110	uint8_t pil;
44219b60	111	uint8_t pif;
d88e784f KJ	112
	113	uint16_t mssrl;
	114	uint32_t mcl;
	115	uint8_t msrc;
	116
	117	bool zoned;
	118	bool cross_zone_read;
	119	uint64_t zone_size_bs;
	120	uint64_t zone_cap_bs;
	121	uint32_t max_active_zones;
	122	uint32_t max_open_zones;
	123	uint32_t zd_extension_size;
e321b4cd KJ	124
	125	uint32_t numzrwa;
	126	uint64_t zrwas;
	127	uint64_t zrwafg;
d88e784f KJ	128	} NvmeNamespaceParams;
	129
	130	typedef struct NvmeNamespace {
	131	DeviceState parent_obj;
	132	BlockConf blkconf;
	133	int32_t bootindex;
	134	int64_t size;
3ef73f94	135	int64_t moff;
d88e784f	136	NvmeIdNs id_ns;
44219b60	137	NvmeIdNsNvm id_ns_nvm;
6146f3dd	138	NvmeLBAF lbaf;
763c05df	139	unsigned int nlbaf;
6146f3dd	140	size_t lbasz;
d88e784f KJ	141	const uint32_t *iocs;
	142	uint8_t csi;
	143	uint16_t status;
	144	int attached;
44219b60	145	uint8_t pif;
d88e784f	146
e321b4cd KJ	147	struct {
	148	uint16_t zrwas;
	149	uint16_t zrwafg;
	150	uint32_t numzrwa;
	151	} zns;
	152
d88e784f KJ	153	QTAILQ_ENTRY(NvmeNamespace) entry;
	154
	155	NvmeIdNsZoned *id_ns_zoned;
	156	NvmeZone *zone_array;
	157	QTAILQ_HEAD(, NvmeZone) exp_open_zones;
	158	QTAILQ_HEAD(, NvmeZone) imp_open_zones;
	159	QTAILQ_HEAD(, NvmeZone) closed_zones;
	160	QTAILQ_HEAD(, NvmeZone) full_zones;
	161	uint32_t num_zones;
	162	uint64_t zone_size;
	163	uint64_t zone_capacity;
	164	uint32_t zone_size_log2;
	165	uint8_t *zd_extensions;
	166	int32_t nr_open_zones;
	167	int32_t nr_active_zones;
	168
	169	NvmeNamespaceParams params;
	170
	171	struct {
	172	uint32_t err_rec;
	173	} features;
	174	} NvmeNamespace;
	175
d88e784f KJ	176	static inline uint32_t nvme_nsid(NvmeNamespace *ns)
	177	{
	178	if (ns) {
	179	return ns->params.nsid;
	180	}
	181
	182	return 0;
	183	}
	184
d88e784f KJ	185	static inline size_t nvme_l2b(NvmeNamespace *ns, uint64_t lba)
d88e784f KJ	186	{
6146f3dd	187	return lba << ns->lbaf.ds;
d88e784f KJ	188	}
	189
	190	static inline size_t nvme_m2b(NvmeNamespace *ns, uint64_t lba)
	191	{
6146f3dd	192	return ns->lbaf.ms * lba;
d88e784f KJ	193	}
d88e784f KJ	194
3ef73f94 KJ	195	static inline int64_t nvme_moff(NvmeNamespace *ns, uint64_t lba)
	196	{
	197	return ns->moff + nvme_m2b(ns, lba);
	198	}
	199
d88e784f KJ	200	static inline bool nvme_ns_ext(NvmeNamespace *ns)
	201	{
	202	return !!NVME_ID_NS_FLBAS_EXTENDED(ns->id_ns.flbas);
	203	}
	204
d88e784f KJ	205	static inline NvmeZoneState nvme_get_zone_state(NvmeZone *zone)
	206	{
	207	return zone->d.zs >> 4;
	208	}
	209
	210	static inline void nvme_set_zone_state(NvmeZone *zone, NvmeZoneState state)
	211	{
	212	zone->d.zs = state << 4;
	213	}
	214
	215	static inline uint64_t nvme_zone_rd_boundary(NvmeNamespace ns, NvmeZone zone)
	216	{
	217	return zone->d.zslba + ns->zone_size;
	218	}
	219
	220	static inline uint64_t nvme_zone_wr_boundary(NvmeZone *zone)
	221	{
	222	return zone->d.zslba + zone->d.zcap;
	223	}
	224
	225	static inline bool nvme_wp_is_valid(NvmeZone *zone)
	226	{
	227	uint8_t st = nvme_get_zone_state(zone);
	228
	229	return st != NVME_ZONE_STATE_FULL &&
	230	st != NVME_ZONE_STATE_READ_ONLY &&
	231	st != NVME_ZONE_STATE_OFFLINE;
	232	}
	233
	234	static inline uint8_t nvme_get_zd_extension(NvmeNamespace ns,
	235	uint32_t zone_idx)
	236	{
	237	return &ns->zd_extensions[zone_idx * ns->params.zd_extension_size];
	238	}
	239
	240	static inline void nvme_aor_inc_open(NvmeNamespace *ns)
	241	{
	242	assert(ns->nr_open_zones >= 0);
	243	if (ns->params.max_open_zones) {
	244	ns->nr_open_zones++;
	245	assert(ns->nr_open_zones <= ns->params.max_open_zones);
	246	}
	247	}
	248
	249	static inline void nvme_aor_dec_open(NvmeNamespace *ns)
	250	{
	251	if (ns->params.max_open_zones) {
	252	assert(ns->nr_open_zones > 0);
	253	ns->nr_open_zones--;
	254	}
	255	assert(ns->nr_open_zones >= 0);
	256	}
	257
	258	static inline void nvme_aor_inc_active(NvmeNamespace *ns)
	259	{
	260	assert(ns->nr_active_zones >= 0);
	261	if (ns->params.max_active_zones) {
	262	ns->nr_active_zones++;
	263	assert(ns->nr_active_zones <= ns->params.max_active_zones);
	264	}
	265	}
	266
	267	static inline void nvme_aor_dec_active(NvmeNamespace *ns)
	268	{
269	if (ns->params.max_active_zones) {
270	assert(ns->nr_active_zones > 0);
271	ns->nr_active_zones--;
272	assert(ns->nr_active_zones >= ns->nr_open_zones);
273	}
274	assert(ns->nr_active_zones >= 0);
275	}
276
277	void nvme_ns_init_format(NvmeNamespace *ns);
5e4f6bcc	278	int nvme_ns_setup(NvmeNamespace ns, Error *errp);
d88e784f KJ	279	void nvme_ns_drain(NvmeNamespace *ns);
	280	void nvme_ns_shutdown(NvmeNamespace *ns);
	281	void nvme_ns_cleanup(NvmeNamespace *ns);
1065abfb	282
f3c507ad	283	typedef struct NvmeAsyncEvent {
5d5a5330	284	QTAILQ_ENTRY(NvmeAsyncEvent) entry;
f3c507ad KB	285	NvmeAerResult result;
	286	} NvmeAsyncEvent;
	287
f80a1c33 KJ	288	enum {
	289	NVME_SG_ALLOC = 1 << 0,
	290	NVME_SG_DMA = 1 << 1,
	291	};
	292
	293	typedef struct NvmeSg {
	294	int flags;
	295
	296	union {
	297	QEMUSGList qsg;
	298	QEMUIOVector iov;
	299	};
	300	} NvmeSg;
	301
d88e784f KJ	302	typedef enum NvmeTxDirection {
	303	NVME_TX_DIRECTION_TO_DEVICE = 0,
	304	NVME_TX_DIRECTION_FROM_DEVICE = 1,
	305	} NvmeTxDirection;
	306
f3c507ad KB	307	typedef struct NvmeRequest {
f3c507ad KB	308	struct NvmeSQueue *sq;
3143df3d	309	struct NvmeNamespace *ns;
7c84b1b8	310	BlockAIOCB *aiocb;
f3c507ad	311	uint16_t status;
2605257a	312	void *opaque;
f3c507ad	313	NvmeCqe cqe;
3143df3d	314	NvmeCmd cmd;
f3c507ad	315	BlockAcctCookie acct;
f80a1c33	316	NvmeSg sg;
f3c507ad KB	317	QTAILQ_ENTRY(NvmeRequest)entry;
	318	} NvmeRequest;
	319
146f720c KJ	320	typedef struct NvmeBounceContext {
	321	NvmeRequest *req;
	322
	323	struct {
	324	QEMUIOVector iov;
	325	uint8_t *bounce;
	326	} data, mdata;
	327	} NvmeBounceContext;
	328
e2f79209 KJ	329	static inline const char *nvme_adm_opc_str(uint8_t opc)
	330	{
	331	switch (opc) {
	332	case NVME_ADM_CMD_DELETE_SQ: return "NVME_ADM_CMD_DELETE_SQ";
	333	case NVME_ADM_CMD_CREATE_SQ: return "NVME_ADM_CMD_CREATE_SQ";
	334	case NVME_ADM_CMD_GET_LOG_PAGE: return "NVME_ADM_CMD_GET_LOG_PAGE";
	335	case NVME_ADM_CMD_DELETE_CQ: return "NVME_ADM_CMD_DELETE_CQ";
	336	case NVME_ADM_CMD_CREATE_CQ: return "NVME_ADM_CMD_CREATE_CQ";
	337	case NVME_ADM_CMD_IDENTIFY: return "NVME_ADM_CMD_IDENTIFY";
	338	case NVME_ADM_CMD_ABORT: return "NVME_ADM_CMD_ABORT";
	339	case NVME_ADM_CMD_SET_FEATURES: return "NVME_ADM_CMD_SET_FEATURES";
	340	case NVME_ADM_CMD_GET_FEATURES: return "NVME_ADM_CMD_GET_FEATURES";
	341	case NVME_ADM_CMD_ASYNC_EV_REQ: return "NVME_ADM_CMD_ASYNC_EV_REQ";
349bf41d	342	case NVME_ADM_CMD_NS_ATTACHMENT: return "NVME_ADM_CMD_NS_ATTACHMENT";
11871f53	343	case NVME_ADM_CMD_VIRT_MNGMT: return "NVME_ADM_CMD_VIRT_MNGMT";
dc04d25e	344	case NVME_ADM_CMD_FORMAT_NVM: return "NVME_ADM_CMD_FORMAT_NVM";
e2f79209 KJ	345	default: return "NVME_ADM_CMD_UNKNOWN";
	346	}
	347	}
	348
	349	static inline const char *nvme_io_opc_str(uint8_t opc)
	350	{
	351	switch (opc) {
	352	case NVME_CMD_FLUSH: return "NVME_NVM_CMD_FLUSH";
	353	case NVME_CMD_WRITE: return "NVME_NVM_CMD_WRITE";
	354	case NVME_CMD_READ: return "NVME_NVM_CMD_READ";
cd42771a	355	case NVME_CMD_COMPARE: return "NVME_NVM_CMD_COMPARE";
e2f79209	356	case NVME_CMD_WRITE_ZEROES: return "NVME_NVM_CMD_WRITE_ZEROES";
2605257a	357	case NVME_CMD_DSM: return "NVME_NVM_CMD_DSM";
3e1da158	358	case NVME_CMD_VERIFY: return "NVME_NVM_CMD_VERIFY";
e4e430b3	359	case NVME_CMD_COPY: return "NVME_NVM_CMD_COPY";
cd42771a KJ	360	case NVME_CMD_ZONE_MGMT_SEND: return "NVME_ZONED_CMD_MGMT_SEND";
	361	case NVME_CMD_ZONE_MGMT_RECV: return "NVME_ZONED_CMD_MGMT_RECV";
	362	case NVME_CMD_ZONE_APPEND: return "NVME_ZONED_CMD_ZONE_APPEND";
e2f79209 KJ	363	default: return "NVME_NVM_CMD_UNKNOWN";
	364	}
	365	}
	366
f3c507ad KB	367	typedef struct NvmeSQueue {
	368	struct NvmeCtrl *ctrl;
	369	uint16_t sqid;
	370	uint16_t cqid;
	371	uint32_t head;
	372	uint32_t tail;
	373	uint32_t size;
	374	uint64_t dma_addr;
	375	QEMUTimer *timer;
	376	NvmeRequest *io_req;
b58deb34 PB	377	QTAILQ_HEAD(, NvmeRequest) req_list;
b58deb34 PB	378	QTAILQ_HEAD(, NvmeRequest) out_req_list;
f3c507ad KB	379	QTAILQ_ENTRY(NvmeSQueue) entry;
	380	} NvmeSQueue;
	381
	382	typedef struct NvmeCQueue {
	383	struct NvmeCtrl *ctrl;
	384	uint8_t phase;
	385	uint16_t cqid;
	386	uint16_t irq_enabled;
	387	uint32_t head;
	388	uint32_t tail;
	389	uint32_t vector;
	390	uint32_t size;
	391	uint64_t dma_addr;
	392	QEMUTimer *timer;
b58deb34 PB	393	QTAILQ_HEAD(, NvmeSQueue) sq_list;
b58deb34 PB	394	QTAILQ_HEAD(, NvmeRequest) req_list;
f3c507ad KB	395	} NvmeCQueue;
f3c507ad KB	396
f3c507ad KB	397	#define TYPE_NVME "nvme"
	398	#define NVME(obj) \
	399	OBJECT_CHECK(NvmeCtrl, (obj), TYPE_NVME)
	400
d88e784f KJ	401	typedef struct NvmeParams {
	402	char *serial;
	403	uint32_t num_queues; /* deprecated since 5.1 */
	404	uint32_t max_ioqpairs;
	405	uint16_t msix_qsize;
	406	uint32_t cmb_size_mb;
	407	uint8_t aerl;
	408	uint32_t aer_max_queued;
	409	uint8_t mdts;
	410	uint8_t vsl;
	411	bool use_intel_id;
	412	uint8_t zasl;
cccc2651	413	bool auto_transition_zones;
d88e784f	414	bool legacy_cmb;
44c2c094	415	uint8_t sriov_max_vfs;
746d42b1 ŁG	416	uint16_t sriov_vq_flexible;
	417	uint16_t sriov_vi_flexible;
	418	uint8_t sriov_max_vq_per_vf;
	419	uint8_t sriov_max_vi_per_vf;
d88e784f	420	} NvmeParams;
46ac29c3	421
f3c507ad KB	422	typedef struct NvmeCtrl {
f3c507ad KB	423	PCIDevice parent_obj;
1901b496	424	MemoryRegion bar0;
f3c507ad KB	425	MemoryRegion iomem;
f3c507ad KB	426	NvmeBar bar;
1065abfb	427	NvmeParams params;
7f0f1ace	428	NvmeBus bus;
f3c507ad	429
e36a261d	430	uint16_t cntlid;
9e7ecdca	431	bool qs_created;
be0677a9	432	uint32_t page_size;
f3c507ad KB	433	uint16_t page_bits;
	434	uint16_t max_prp_ents;
	435	uint16_t cqe_size;
	436	uint16_t sqe_size;
f3c507ad	437	uint32_t max_q_ents;
5d5a5330	438	uint8_t outstanding_aers;
ca247d35	439	uint32_t irq_status;
83d7ed5c	440	int cq_pending;
3036a626 KH	441	uint64_t host_timestamp; /* Timestamp sent by the host */
3036a626 KH	442	uint64_t timestamp_set_qemu_clock_ms; /* QEMU clock time */
94a7897c KJ	443	uint64_t starttime_ms;
94a7897c KJ	444	uint16_t temperature;
4714791b	445	uint8_t smart_critical_warning;
decc0261 ŁG	446	uint32_t conf_msix_qsize;
decc0261 ŁG	447	uint32_t conf_ioqpairs;
f3c507ad	448
f4319477 PK	449	struct {
	450	MemoryRegion mem;
	451	uint8_t *buf;
	452	bool cmse;
	453	hwaddr cba;
	454	} cmb;
	455
7ec9f2ee NN	456	struct {
	457	HostMemoryBackend *dev;
	458	bool cmse;
	459	hwaddr cba;
	460	} pmr;
6cf94132	461
5d5a5330 KJ	462	uint8_t aer_mask;
	463	NvmeRequest **aer_reqs;
	464	QTAILQ_HEAD(, NvmeAsyncEvent) aer_queue;
	465	int aer_queued;
	466
67ce28a1 GA	467	uint32_t dmrsl;
67ce28a1 GA	468
f432fdfa MI	469	/* Namespace ID is started with 1 so bitmap should be 1-based */
	470	#define NVME_CHANGED_NSID_SIZE (NVME_MAX_NAMESPACES + 1)
	471	DECLARE_BITMAP(changed_nsids, NVME_CHANGED_NSID_SIZE);
	472
982ed66b MI	473	NvmeSubsystem *subsys;
982ed66b MI	474
7f0f1ace	475	NvmeNamespace namespace;
72ea5c2c	476	NvmeNamespace *namespaces[NVME_MAX_NAMESPACES + 1];
f3c507ad KB	477	NvmeSQueue **sq;
	478	NvmeCQueue **cq;
	479	NvmeSQueue admin_sq;
	480	NvmeCQueue admin_cq;
	481	NvmeIdCtrl id_ctrl;
d88e784f KJ	482
	483	struct {
	484	struct {
	485	uint16_t temp_thresh_hi;
	486	uint16_t temp_thresh_low;
	487	};
d0c0697b NN	488
	489	uint32_t async_config;
	490	NvmeHostBehaviorSupport hbs;
d88e784f	491	} features;
5e6f963f LM	492
5e6f963f LM	493	NvmePriCtrlCap pri_ctrl_cap;
99f48ae7	494	NvmeSecCtrlList sec_ctrl_list;
11871f53 ŁG	495	struct {
	496	uint16_t vqrfap;
	497	uint16_t virfap;
	498	} next_pri_ctrl_cap; /* These override pri_ctrl_cap after reset */
f3c507ad KB	499	} NvmeCtrl;
f3c507ad KB	500
1e9c685e ŁG	501	typedef enum NvmeResetType {
	502	NVME_RESET_FUNCTION = 0,
	503	NVME_RESET_CONTROLLER = 1,
	504	} NvmeResetType;
	505
7f0f1ace	506	static inline NvmeNamespace nvme_ns(NvmeCtrl n, uint32_t nsid)
3adee1c2	507	{
d88e784f	508	if (!nsid \|\| nsid > NVME_MAX_NAMESPACES) {
7f0f1ace KJ	509	return NULL;
	510	}
	511
72ea5c2c	512	return n->namespaces[nsid];
3adee1c2 KJ	513	}
3adee1c2 KJ	514
6a09a3d7 KJ	515	static inline NvmeCQueue nvme_cq(NvmeRequest req)
	516	{
	517	NvmeSQueue *sq = req->sq;
	518	NvmeCtrl *n = sq->ctrl;
	519
	520	return n->cq[sq->cqid];
	521	}
	522
	523	static inline NvmeCtrl nvme_ctrl(NvmeRequest req)
	524	{
	525	NvmeSQueue *sq = req->sq;
	526	return sq->ctrl;
	527	}
	528
146f720c KJ	529	static inline uint16_t nvme_cid(NvmeRequest *req)
	530	{
	531	if (!req) {
	532	return 0xffff;
	533	}
	534
	535	return le16_to_cpu(req->cqe.cid);
	536	}
	537
99f48ae7 LM	538	static inline NvmeSecCtrlEntry nvme_sctrl(NvmeCtrl n)
	539	{
	540	PCIDevice *pci_dev = &n->parent_obj;
	541	NvmeCtrl *pf = NVME(pcie_sriov_get_pf(pci_dev));
	542
	543	if (pci_is_vf(pci_dev)) {
	544	return &pf->sec_ctrl_list.sec[pcie_sriov_vf_number(pci_dev)];
	545	}
	546
	547	return NULL;
	548	}
	549
11871f53 ŁG	550	static inline NvmeSecCtrlEntry nvme_sctrl_for_cntlid(NvmeCtrl n,
	551	uint16_t cntlid)
	552	{
	553	NvmeSecCtrlList *list = &n->sec_ctrl_list;
	554	uint8_t i;
	555
	556	for (i = 0; i < list->numcntl; i++) {
	557	if (le16_to_cpu(list->sec[i].scid) == cntlid) {
	558	return &list->sec[i];
	559	}
	560	}
	561
	562	return NULL;
	563	}
	564
e5489356	565	void nvme_attach_ns(NvmeCtrl n, NvmeNamespace ns);
8d3a17be	566	uint16_t nvme_bounce_data(NvmeCtrl n, void ptr, uint32_t len,
146f720c	567	NvmeTxDirection dir, NvmeRequest *req);
8d3a17be	568	uint16_t nvme_bounce_mdata(NvmeCtrl n, void ptr, uint32_t len,
146f720c KJ	569	NvmeTxDirection dir, NvmeRequest *req);
	570	void nvme_rw_complete_cb(void *opaque, int ret);
	571	uint16_t nvme_map_dptr(NvmeCtrl n, NvmeSg sg, size_t len,
	572	NvmeCmd *cmd);
7f0f1ace	573
52581c71	574	#endif /* HW_NVME_NVME_H */