2 * QEMU NVM Express Controller
4 * Copyright (c) 2012, Intel Corporation
6 * Written by Keith Busch <keith.busch@intel.com>
8 * This code is licensed under the GNU GPL v2 or later.
12 * Reference Specs: http://www.nvmexpress.org, 1.4, 1.3, 1.2, 1.1, 1.0e
14 * https://nvmexpress.org/developers/nvme-specification/
17 * Notes on coding style
18 * ---------------------
19 * While QEMU coding style prefers lowercase hexadecimals in constants, the
20 * NVMe subsystem use thes format from the NVMe specifications in the comments
21 * (i.e. 'h' suffix instead of '0x' prefix).
25 * See docs/system/nvme.rst for extensive documentation.
28 * -drive file=<file>,if=none,id=<drive_id>
29 * -device nvme-subsys,id=<subsys_id>,nqn=<nqn_id>
30 * -device nvme,serial=<serial>,id=<bus_name>, \
31 * cmb_size_mb=<cmb_size_mb[optional]>, \
32 * [pmrdev=<mem_backend_file_id>,] \
33 * max_ioqpairs=<N[optional]>, \
34 * aerl=<N[optional]>,aer_max_queued=<N[optional]>, \
35 * mdts=<N[optional]>,vsl=<N[optional]>, \
36 * zoned.zasl=<N[optional]>, \
37 * zoned.auto_transition=<on|off[optional]>, \
38 * sriov_max_vfs=<N[optional]> \
39 * sriov_vq_flexible=<N[optional]> \
40 * sriov_vi_flexible=<N[optional]> \
41 * sriov_max_vi_per_vf=<N[optional]> \
42 * sriov_max_vq_per_vf=<N[optional]> \
44 * -device nvme-ns,drive=<drive_id>,bus=<bus_name>,nsid=<nsid>,\
45 * zoned=<true|false[optional]>, \
46 * subsys=<subsys_id>,detached=<true|false[optional]>
48 * Note cmb_size_mb denotes size of CMB in MB. CMB is assumed to be at
49 * offset 0 in BAR2 and supports only WDS, RDS and SQS for now. By default, the
50 * device will use the "v1.4 CMB scheme" - use the `legacy-cmb` parameter to
51 * always enable the CMBLOC and CMBSZ registers (v1.3 behavior).
53 * Enabling pmr emulation can be achieved by pointing to memory-backend-file.
55 * -object memory-backend-file,id=<mem_id>,share=on,mem-path=<file_path>, \
56 * size=<size> .... -device nvme,...,pmrdev=<mem_id>
58 * The PMR will use BAR 4/5 exclusively.
60 * To place controller(s) and namespace(s) to a subsystem, then provide
61 * nvme-subsys device as above.
63 * nvme subsystem device parameters
64 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
66 * This parameter provides the `<nqn_id>` part of the string
67 * `nqn.2019-08.org.qemu:<nqn_id>` which will be reported in the SUBNQN field
68 * of subsystem controllers. Note that `<nqn_id>` should be unique per
69 * subsystem, but this is not enforced by QEMU. If not specified, it will
70 * default to the value of the `id` parameter (`<subsys_id>`).
72 * nvme device parameters
73 * ~~~~~~~~~~~~~~~~~~~~~~
75 * Specifying this parameter attaches the controller to the subsystem and
76 * the SUBNQN field in the controller will report the NQN of the subsystem
77 * device. This also enables multi controller capability represented in
78 * Identify Controller data structure in CMIC (Controller Multi-path I/O and
79 * Namespace Sharing Capabilities).
82 * The Asynchronous Event Request Limit (AERL). Indicates the maximum number
83 * of concurrently outstanding Asynchronous Event Request commands support
84 * by the controller. This is a 0's based value.
87 * This is the maximum number of events that the device will enqueue for
88 * completion when there are no outstanding AERs. When the maximum number of
89 * enqueued events are reached, subsequent events will be dropped.
92 * Indicates the maximum data transfer size for a command that transfers data
93 * between host-accessible memory and the controller. The value is specified
94 * as a power of two (2^n) and is in units of the minimum memory page size
95 * (CAP.MPSMIN). The default value is 7 (i.e. 512 KiB).
98 * Indicates the maximum data size limit for the Verify command. Like `mdts`,
99 * this value is specified as a power of two (2^n) and is in units of the
100 * minimum memory page size (CAP.MPSMIN). The default value is 7 (i.e. 512
104 * Indicates the maximum data transfer size for the Zone Append command. Like
105 * `mdts`, the value is specified as a power of two (2^n) and is in units of
106 * the minimum memory page size (CAP.MPSMIN). The default value is 0 (i.e.
107 * defaulting to the value of `mdts`).
109 * - `zoned.auto_transition`
110 * Indicates if zones in zone state implicitly opened can be automatically
111 * transitioned to zone state closed for resource management purposes.
115 * Indicates the maximum number of PCIe virtual functions supported
116 * by the controller. The default value is 0. Specifying a non-zero value
117 * enables reporting of both SR-IOV and ARI capabilities by the NVMe device.
118 * Virtual function controllers will not report SR-IOV capability.
120 * NOTE: Single Root I/O Virtualization support is experimental.
121 * All the related parameters may be subject to change.
123 * - `sriov_vq_flexible`
124 * Indicates the total number of flexible queue resources assignable to all
125 * the secondary controllers. Implicitly sets the number of primary
126 * controller's private resources to `(max_ioqpairs - sriov_vq_flexible)`.
128 * - `sriov_vi_flexible`
129 * Indicates the total number of flexible interrupt resources assignable to
130 * all the secondary controllers. Implicitly sets the number of primary
131 * controller's private resources to `(msix_qsize - sriov_vi_flexible)`.
133 * - `sriov_max_vi_per_vf`
134 * Indicates the maximum number of virtual interrupt resources assignable
135 * to a secondary controller. The default 0 resolves to
136 * `(sriov_vi_flexible / sriov_max_vfs)`.
138 * - `sriov_max_vq_per_vf`
139 * Indicates the maximum number of virtual queue resources assignable to
140 * a secondary controller. The default 0 resolves to
141 * `(sriov_vq_flexible / sriov_max_vfs)`.
143 * nvme namespace device parameters
144 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
146 * When the parent nvme device (as defined explicitly by the 'bus' parameter
147 * or implicitly by the most recently defined NvmeBus) is linked to an
148 * nvme-subsys device, the namespace will be attached to all controllers in
149 * the subsystem. If set to 'off' (the default), the namespace will remain a
150 * private namespace and may only be attached to a single controller at a
154 * This parameter is only valid together with the `subsys` parameter. If left
155 * at the default value (`false/off`), the namespace will be attached to all
156 * controllers in the NVMe subsystem at boot-up. If set to `true/on`, the
157 * namespace will be available in the subsystem but not attached to any
160 * Setting `zoned` to true selects Zoned Command Set at the namespace.
161 * In this case, the following namespace properties are available to configure
163 * zoned.zone_size=<zone size in bytes, default: 128MiB>
164 * The number may be followed by K, M, G as in kilo-, mega- or giga-.
166 * zoned.zone_capacity=<zone capacity in bytes, default: zone size>
167 * The value 0 (default) forces zone capacity to be the same as zone
168 * size. The value of this property may not exceed zone size.
170 * zoned.descr_ext_size=<zone descriptor extension size, default 0>
171 * This value needs to be specified in 64B units. If it is zero,
172 * namespace(s) will not support zone descriptor extensions.
174 * zoned.max_active=<Maximum Active Resources (zones), default: 0>
175 * The default value means there is no limit to the number of
176 * concurrently active zones.
178 * zoned.max_open=<Maximum Open Resources (zones), default: 0>
179 * The default value means there is no limit to the number of
180 * concurrently open zones.
182 * zoned.cross_read=<enable RAZB, default: false>
183 * Setting this property to true enables Read Across Zone Boundaries.
186 #include "qemu/osdep.h"
187 #include "qemu/cutils.h"
188 #include "qemu/error-report.h"
189 #include "qemu/log.h"
190 #include "qemu/units.h"
191 #include "qemu/range.h"
192 #include "qapi/error.h"
193 #include "qapi/visitor.h"
194 #include "sysemu/sysemu.h"
195 #include "sysemu/block-backend.h"
196 #include "sysemu/hostmem.h"
197 #include "hw/pci/msix.h"
198 #include "hw/pci/pcie_sriov.h"
199 #include "migration/vmstate.h"
205 #define NVME_MAX_IOQPAIRS 0xffff
206 #define NVME_DB_SIZE 4
207 #define NVME_SPEC_VER 0x00010400
208 #define NVME_CMB_BIR 2
209 #define NVME_PMR_BIR 4
210 #define NVME_TEMPERATURE 0x143
211 #define NVME_TEMPERATURE_WARNING 0x157
212 #define NVME_TEMPERATURE_CRITICAL 0x175
213 #define NVME_NUM_FW_SLOTS 1
214 #define NVME_DEFAULT_MAX_ZA_SIZE (128 * KiB)
215 #define NVME_MAX_VFS 127
216 #define NVME_VF_RES_GRANULARITY 1
217 #define NVME_VF_OFFSET 0x1
218 #define NVME_VF_STRIDE 1
220 #define NVME_GUEST_ERR(trace, fmt, ...) \
222 (trace_##trace)(__VA_ARGS__); \
223 qemu_log_mask(LOG_GUEST_ERROR, #trace \
224 " in %s: " fmt "\n", __func__, ## __VA_ARGS__); \
227 static const bool nvme_feature_support
[NVME_FID_MAX
] = {
228 [NVME_ARBITRATION
] = true,
229 [NVME_POWER_MANAGEMENT
] = true,
230 [NVME_TEMPERATURE_THRESHOLD
] = true,
231 [NVME_ERROR_RECOVERY
] = true,
232 [NVME_VOLATILE_WRITE_CACHE
] = true,
233 [NVME_NUMBER_OF_QUEUES
] = true,
234 [NVME_INTERRUPT_COALESCING
] = true,
235 [NVME_INTERRUPT_VECTOR_CONF
] = true,
236 [NVME_WRITE_ATOMICITY
] = true,
237 [NVME_ASYNCHRONOUS_EVENT_CONF
] = true,
238 [NVME_TIMESTAMP
] = true,
239 [NVME_HOST_BEHAVIOR_SUPPORT
] = true,
240 [NVME_COMMAND_SET_PROFILE
] = true,
243 static const uint32_t nvme_feature_cap
[NVME_FID_MAX
] = {
244 [NVME_TEMPERATURE_THRESHOLD
] = NVME_FEAT_CAP_CHANGE
,
245 [NVME_ERROR_RECOVERY
] = NVME_FEAT_CAP_CHANGE
| NVME_FEAT_CAP_NS
,
246 [NVME_VOLATILE_WRITE_CACHE
] = NVME_FEAT_CAP_CHANGE
,
247 [NVME_NUMBER_OF_QUEUES
] = NVME_FEAT_CAP_CHANGE
,
248 [NVME_ASYNCHRONOUS_EVENT_CONF
] = NVME_FEAT_CAP_CHANGE
,
249 [NVME_TIMESTAMP
] = NVME_FEAT_CAP_CHANGE
,
250 [NVME_HOST_BEHAVIOR_SUPPORT
] = NVME_FEAT_CAP_CHANGE
,
251 [NVME_COMMAND_SET_PROFILE
] = NVME_FEAT_CAP_CHANGE
,
254 static const uint32_t nvme_cse_acs
[256] = {
255 [NVME_ADM_CMD_DELETE_SQ
] = NVME_CMD_EFF_CSUPP
,
256 [NVME_ADM_CMD_CREATE_SQ
] = NVME_CMD_EFF_CSUPP
,
257 [NVME_ADM_CMD_GET_LOG_PAGE
] = NVME_CMD_EFF_CSUPP
,
258 [NVME_ADM_CMD_DELETE_CQ
] = NVME_CMD_EFF_CSUPP
,
259 [NVME_ADM_CMD_CREATE_CQ
] = NVME_CMD_EFF_CSUPP
,
260 [NVME_ADM_CMD_IDENTIFY
] = NVME_CMD_EFF_CSUPP
,
261 [NVME_ADM_CMD_ABORT
] = NVME_CMD_EFF_CSUPP
,
262 [NVME_ADM_CMD_SET_FEATURES
] = NVME_CMD_EFF_CSUPP
,
263 [NVME_ADM_CMD_GET_FEATURES
] = NVME_CMD_EFF_CSUPP
,
264 [NVME_ADM_CMD_ASYNC_EV_REQ
] = NVME_CMD_EFF_CSUPP
,
265 [NVME_ADM_CMD_NS_ATTACHMENT
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_NIC
,
266 [NVME_ADM_CMD_VIRT_MNGMT
] = NVME_CMD_EFF_CSUPP
,
267 [NVME_ADM_CMD_DBBUF_CONFIG
] = NVME_CMD_EFF_CSUPP
,
268 [NVME_ADM_CMD_FORMAT_NVM
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
271 static const uint32_t nvme_cse_iocs_none
[256];
273 static const uint32_t nvme_cse_iocs_nvm
[256] = {
274 [NVME_CMD_FLUSH
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
275 [NVME_CMD_WRITE_ZEROES
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
276 [NVME_CMD_WRITE
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
277 [NVME_CMD_READ
] = NVME_CMD_EFF_CSUPP
,
278 [NVME_CMD_DSM
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
279 [NVME_CMD_VERIFY
] = NVME_CMD_EFF_CSUPP
,
280 [NVME_CMD_COPY
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
281 [NVME_CMD_COMPARE
] = NVME_CMD_EFF_CSUPP
,
284 static const uint32_t nvme_cse_iocs_zoned
[256] = {
285 [NVME_CMD_FLUSH
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
286 [NVME_CMD_WRITE_ZEROES
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
287 [NVME_CMD_WRITE
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
288 [NVME_CMD_READ
] = NVME_CMD_EFF_CSUPP
,
289 [NVME_CMD_DSM
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
290 [NVME_CMD_VERIFY
] = NVME_CMD_EFF_CSUPP
,
291 [NVME_CMD_COPY
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
292 [NVME_CMD_COMPARE
] = NVME_CMD_EFF_CSUPP
,
293 [NVME_CMD_ZONE_APPEND
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
294 [NVME_CMD_ZONE_MGMT_SEND
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
295 [NVME_CMD_ZONE_MGMT_RECV
] = NVME_CMD_EFF_CSUPP
,
298 static void nvme_process_sq(void *opaque
);
299 static void nvme_ctrl_reset(NvmeCtrl
*n
, NvmeResetType rst
);
301 static uint16_t nvme_sqid(NvmeRequest
*req
)
303 return le16_to_cpu(req
->sq
->sqid
);
306 static void nvme_assign_zone_state(NvmeNamespace
*ns
, NvmeZone
*zone
,
309 if (QTAILQ_IN_USE(zone
, entry
)) {
310 switch (nvme_get_zone_state(zone
)) {
311 case NVME_ZONE_STATE_EXPLICITLY_OPEN
:
312 QTAILQ_REMOVE(&ns
->exp_open_zones
, zone
, entry
);
314 case NVME_ZONE_STATE_IMPLICITLY_OPEN
:
315 QTAILQ_REMOVE(&ns
->imp_open_zones
, zone
, entry
);
317 case NVME_ZONE_STATE_CLOSED
:
318 QTAILQ_REMOVE(&ns
->closed_zones
, zone
, entry
);
320 case NVME_ZONE_STATE_FULL
:
321 QTAILQ_REMOVE(&ns
->full_zones
, zone
, entry
);
327 nvme_set_zone_state(zone
, state
);
330 case NVME_ZONE_STATE_EXPLICITLY_OPEN
:
331 QTAILQ_INSERT_TAIL(&ns
->exp_open_zones
, zone
, entry
);
333 case NVME_ZONE_STATE_IMPLICITLY_OPEN
:
334 QTAILQ_INSERT_TAIL(&ns
->imp_open_zones
, zone
, entry
);
336 case NVME_ZONE_STATE_CLOSED
:
337 QTAILQ_INSERT_TAIL(&ns
->closed_zones
, zone
, entry
);
339 case NVME_ZONE_STATE_FULL
:
340 QTAILQ_INSERT_TAIL(&ns
->full_zones
, zone
, entry
);
341 case NVME_ZONE_STATE_READ_ONLY
:
348 static uint16_t nvme_zns_check_resources(NvmeNamespace
*ns
, uint32_t act
,
349 uint32_t opn
, uint32_t zrwa
)
351 if (ns
->params
.max_active_zones
!= 0 &&
352 ns
->nr_active_zones
+ act
> ns
->params
.max_active_zones
) {
353 trace_pci_nvme_err_insuff_active_res(ns
->params
.max_active_zones
);
354 return NVME_ZONE_TOO_MANY_ACTIVE
| NVME_DNR
;
357 if (ns
->params
.max_open_zones
!= 0 &&
358 ns
->nr_open_zones
+ opn
> ns
->params
.max_open_zones
) {
359 trace_pci_nvme_err_insuff_open_res(ns
->params
.max_open_zones
);
360 return NVME_ZONE_TOO_MANY_OPEN
| NVME_DNR
;
363 if (zrwa
> ns
->zns
.numzrwa
) {
364 return NVME_NOZRWA
| NVME_DNR
;
371 * Check if we can open a zone without exceeding open/active limits.
372 * AOR stands for "Active and Open Resources" (see TP 4053 section 2.5).
374 static uint16_t nvme_aor_check(NvmeNamespace
*ns
, uint32_t act
, uint32_t opn
)
376 return nvme_zns_check_resources(ns
, act
, opn
, 0);
379 static bool nvme_addr_is_cmb(NvmeCtrl
*n
, hwaddr addr
)
387 lo
= n
->params
.legacy_cmb
? n
->cmb
.mem
.addr
: n
->cmb
.cba
;
388 hi
= lo
+ int128_get64(n
->cmb
.mem
.size
);
390 return addr
>= lo
&& addr
< hi
;
393 static inline void *nvme_addr_to_cmb(NvmeCtrl
*n
, hwaddr addr
)
395 hwaddr base
= n
->params
.legacy_cmb
? n
->cmb
.mem
.addr
: n
->cmb
.cba
;
396 return &n
->cmb
.buf
[addr
- base
];
399 static bool nvme_addr_is_pmr(NvmeCtrl
*n
, hwaddr addr
)
407 hi
= n
->pmr
.cba
+ int128_get64(n
->pmr
.dev
->mr
.size
);
409 return addr
>= n
->pmr
.cba
&& addr
< hi
;
412 static inline void *nvme_addr_to_pmr(NvmeCtrl
*n
, hwaddr addr
)
414 return memory_region_get_ram_ptr(&n
->pmr
.dev
->mr
) + (addr
- n
->pmr
.cba
);
417 static inline bool nvme_addr_is_iomem(NvmeCtrl
*n
, hwaddr addr
)
422 * The purpose of this check is to guard against invalid "local" access to
423 * the iomem (i.e. controller registers). Thus, we check against the range
424 * covered by the 'bar0' MemoryRegion since that is currently composed of
425 * two subregions (the NVMe "MBAR" and the MSI-X table/pba). Note, however,
426 * that if the device model is ever changed to allow the CMB to be located
427 * in BAR0 as well, then this must be changed.
430 hi
= lo
+ int128_get64(n
->bar0
.size
);
432 return addr
>= lo
&& addr
< hi
;
435 static int nvme_addr_read(NvmeCtrl
*n
, hwaddr addr
, void *buf
, int size
)
437 hwaddr hi
= addr
+ size
- 1;
442 if (n
->bar
.cmbsz
&& nvme_addr_is_cmb(n
, addr
) && nvme_addr_is_cmb(n
, hi
)) {
443 memcpy(buf
, nvme_addr_to_cmb(n
, addr
), size
);
447 if (nvme_addr_is_pmr(n
, addr
) && nvme_addr_is_pmr(n
, hi
)) {
448 memcpy(buf
, nvme_addr_to_pmr(n
, addr
), size
);
452 return pci_dma_read(PCI_DEVICE(n
), addr
, buf
, size
);
455 static int nvme_addr_write(NvmeCtrl
*n
, hwaddr addr
, const void *buf
, int size
)
457 hwaddr hi
= addr
+ size
- 1;
462 if (n
->bar
.cmbsz
&& nvme_addr_is_cmb(n
, addr
) && nvme_addr_is_cmb(n
, hi
)) {
463 memcpy(nvme_addr_to_cmb(n
, addr
), buf
, size
);
467 if (nvme_addr_is_pmr(n
, addr
) && nvme_addr_is_pmr(n
, hi
)) {
468 memcpy(nvme_addr_to_pmr(n
, addr
), buf
, size
);
472 return pci_dma_write(PCI_DEVICE(n
), addr
, buf
, size
);
475 static bool nvme_nsid_valid(NvmeCtrl
*n
, uint32_t nsid
)
478 (nsid
== NVME_NSID_BROADCAST
|| nsid
<= NVME_MAX_NAMESPACES
);
481 static int nvme_check_sqid(NvmeCtrl
*n
, uint16_t sqid
)
483 return sqid
< n
->conf_ioqpairs
+ 1 && n
->sq
[sqid
] != NULL
? 0 : -1;
486 static int nvme_check_cqid(NvmeCtrl
*n
, uint16_t cqid
)
488 return cqid
< n
->conf_ioqpairs
+ 1 && n
->cq
[cqid
] != NULL
? 0 : -1;
491 static void nvme_inc_cq_tail(NvmeCQueue
*cq
)
494 if (cq
->tail
>= cq
->size
) {
496 cq
->phase
= !cq
->phase
;
500 static void nvme_inc_sq_head(NvmeSQueue
*sq
)
502 sq
->head
= (sq
->head
+ 1) % sq
->size
;
505 static uint8_t nvme_cq_full(NvmeCQueue
*cq
)
507 return (cq
->tail
+ 1) % cq
->size
== cq
->head
;
510 static uint8_t nvme_sq_empty(NvmeSQueue
*sq
)
512 return sq
->head
== sq
->tail
;
515 static void nvme_irq_check(NvmeCtrl
*n
)
517 PCIDevice
*pci
= PCI_DEVICE(n
);
518 uint32_t intms
= ldl_le_p(&n
->bar
.intms
);
520 if (msix_enabled(pci
)) {
523 if (~intms
& n
->irq_status
) {
526 pci_irq_deassert(pci
);
530 static void nvme_irq_assert(NvmeCtrl
*n
, NvmeCQueue
*cq
)
532 PCIDevice
*pci
= PCI_DEVICE(n
);
534 if (cq
->irq_enabled
) {
535 if (msix_enabled(pci
)) {
536 trace_pci_nvme_irq_msix(cq
->vector
);
537 msix_notify(pci
, cq
->vector
);
539 trace_pci_nvme_irq_pin();
540 assert(cq
->vector
< 32);
541 n
->irq_status
|= 1 << cq
->vector
;
545 trace_pci_nvme_irq_masked();
549 static void nvme_irq_deassert(NvmeCtrl
*n
, NvmeCQueue
*cq
)
551 if (cq
->irq_enabled
) {
552 if (msix_enabled(PCI_DEVICE(n
))) {
555 assert(cq
->vector
< 32);
556 if (!n
->cq_pending
) {
557 n
->irq_status
&= ~(1 << cq
->vector
);
564 static void nvme_req_clear(NvmeRequest
*req
)
569 memset(&req
->cqe
, 0x0, sizeof(req
->cqe
));
570 req
->status
= NVME_SUCCESS
;
573 static inline void nvme_sg_init(NvmeCtrl
*n
, NvmeSg
*sg
, bool dma
)
576 pci_dma_sglist_init(&sg
->qsg
, PCI_DEVICE(n
), 0);
577 sg
->flags
= NVME_SG_DMA
;
579 qemu_iovec_init(&sg
->iov
, 0);
582 sg
->flags
|= NVME_SG_ALLOC
;
585 static inline void nvme_sg_unmap(NvmeSg
*sg
)
587 if (!(sg
->flags
& NVME_SG_ALLOC
)) {
591 if (sg
->flags
& NVME_SG_DMA
) {
592 qemu_sglist_destroy(&sg
->qsg
);
594 qemu_iovec_destroy(&sg
->iov
);
597 memset(sg
, 0x0, sizeof(*sg
));
601 * When metadata is transfered as extended LBAs, the DPTR mapped into `sg`
602 * holds both data and metadata. This function splits the data and metadata
603 * into two separate QSG/IOVs.
605 static void nvme_sg_split(NvmeSg
*sg
, NvmeNamespace
*ns
, NvmeSg
*data
,
609 uint32_t trans_len
, count
= ns
->lbasz
;
611 bool dma
= sg
->flags
& NVME_SG_DMA
;
613 size_t sg_len
= dma
? sg
->qsg
.size
: sg
->iov
.size
;
616 assert(sg
->flags
& NVME_SG_ALLOC
);
619 sge_len
= dma
? sg
->qsg
.sg
[sg_idx
].len
: sg
->iov
.iov
[sg_idx
].iov_len
;
621 trans_len
= MIN(sg_len
, count
);
622 trans_len
= MIN(trans_len
, sge_len
- offset
);
626 qemu_sglist_add(&dst
->qsg
, sg
->qsg
.sg
[sg_idx
].base
+ offset
,
629 qemu_iovec_add(&dst
->iov
,
630 sg
->iov
.iov
[sg_idx
].iov_base
+ offset
,
640 dst
= (dst
== data
) ? mdata
: data
;
641 count
= (dst
== data
) ? ns
->lbasz
: ns
->lbaf
.ms
;
644 if (sge_len
== offset
) {
651 static uint16_t nvme_map_addr_cmb(NvmeCtrl
*n
, QEMUIOVector
*iov
, hwaddr addr
,
658 trace_pci_nvme_map_addr_cmb(addr
, len
);
660 if (!nvme_addr_is_cmb(n
, addr
) || !nvme_addr_is_cmb(n
, addr
+ len
- 1)) {
661 return NVME_DATA_TRAS_ERROR
;
664 qemu_iovec_add(iov
, nvme_addr_to_cmb(n
, addr
), len
);
669 static uint16_t nvme_map_addr_pmr(NvmeCtrl
*n
, QEMUIOVector
*iov
, hwaddr addr
,
676 if (!nvme_addr_is_pmr(n
, addr
) || !nvme_addr_is_pmr(n
, addr
+ len
- 1)) {
677 return NVME_DATA_TRAS_ERROR
;
680 qemu_iovec_add(iov
, nvme_addr_to_pmr(n
, addr
), len
);
685 static uint16_t nvme_map_addr(NvmeCtrl
*n
, NvmeSg
*sg
, hwaddr addr
, size_t len
)
687 bool cmb
= false, pmr
= false;
693 trace_pci_nvme_map_addr(addr
, len
);
695 if (nvme_addr_is_iomem(n
, addr
)) {
696 return NVME_DATA_TRAS_ERROR
;
699 if (nvme_addr_is_cmb(n
, addr
)) {
701 } else if (nvme_addr_is_pmr(n
, addr
)) {
706 if (sg
->flags
& NVME_SG_DMA
) {
707 return NVME_INVALID_USE_OF_CMB
| NVME_DNR
;
710 if (sg
->iov
.niov
+ 1 > IOV_MAX
) {
711 goto max_mappings_exceeded
;
715 return nvme_map_addr_cmb(n
, &sg
->iov
, addr
, len
);
717 return nvme_map_addr_pmr(n
, &sg
->iov
, addr
, len
);
721 if (!(sg
->flags
& NVME_SG_DMA
)) {
722 return NVME_INVALID_USE_OF_CMB
| NVME_DNR
;
725 if (sg
->qsg
.nsg
+ 1 > IOV_MAX
) {
726 goto max_mappings_exceeded
;
729 qemu_sglist_add(&sg
->qsg
, addr
, len
);
733 max_mappings_exceeded
:
734 NVME_GUEST_ERR(pci_nvme_ub_too_many_mappings
,
735 "number of mappings exceed 1024");
736 return NVME_INTERNAL_DEV_ERROR
| NVME_DNR
;
739 static inline bool nvme_addr_is_dma(NvmeCtrl
*n
, hwaddr addr
)
741 return !(nvme_addr_is_cmb(n
, addr
) || nvme_addr_is_pmr(n
, addr
));
744 static uint16_t nvme_map_prp(NvmeCtrl
*n
, NvmeSg
*sg
, uint64_t prp1
,
745 uint64_t prp2
, uint32_t len
)
747 hwaddr trans_len
= n
->page_size
- (prp1
% n
->page_size
);
748 trans_len
= MIN(len
, trans_len
);
749 int num_prps
= (len
>> n
->page_bits
) + 1;
753 trace_pci_nvme_map_prp(trans_len
, len
, prp1
, prp2
, num_prps
);
755 nvme_sg_init(n
, sg
, nvme_addr_is_dma(n
, prp1
));
757 status
= nvme_map_addr(n
, sg
, prp1
, trans_len
);
764 if (len
> n
->page_size
) {
765 uint64_t prp_list
[n
->max_prp_ents
];
766 uint32_t nents
, prp_trans
;
770 * The first PRP list entry, pointed to by PRP2 may contain offset.
771 * Hence, we need to calculate the number of entries in based on
774 nents
= (n
->page_size
- (prp2
& (n
->page_size
- 1))) >> 3;
775 prp_trans
= MIN(n
->max_prp_ents
, nents
) * sizeof(uint64_t);
776 ret
= nvme_addr_read(n
, prp2
, (void *)prp_list
, prp_trans
);
778 trace_pci_nvme_err_addr_read(prp2
);
779 status
= NVME_DATA_TRAS_ERROR
;
783 uint64_t prp_ent
= le64_to_cpu(prp_list
[i
]);
785 if (i
== nents
- 1 && len
> n
->page_size
) {
786 if (unlikely(prp_ent
& (n
->page_size
- 1))) {
787 trace_pci_nvme_err_invalid_prplist_ent(prp_ent
);
788 status
= NVME_INVALID_PRP_OFFSET
| NVME_DNR
;
793 nents
= (len
+ n
->page_size
- 1) >> n
->page_bits
;
794 nents
= MIN(nents
, n
->max_prp_ents
);
795 prp_trans
= nents
* sizeof(uint64_t);
796 ret
= nvme_addr_read(n
, prp_ent
, (void *)prp_list
,
799 trace_pci_nvme_err_addr_read(prp_ent
);
800 status
= NVME_DATA_TRAS_ERROR
;
803 prp_ent
= le64_to_cpu(prp_list
[i
]);
806 if (unlikely(prp_ent
& (n
->page_size
- 1))) {
807 trace_pci_nvme_err_invalid_prplist_ent(prp_ent
);
808 status
= NVME_INVALID_PRP_OFFSET
| NVME_DNR
;
812 trans_len
= MIN(len
, n
->page_size
);
813 status
= nvme_map_addr(n
, sg
, prp_ent
, trans_len
);
822 if (unlikely(prp2
& (n
->page_size
- 1))) {
823 trace_pci_nvme_err_invalid_prp2_align(prp2
);
824 status
= NVME_INVALID_PRP_OFFSET
| NVME_DNR
;
827 status
= nvme_map_addr(n
, sg
, prp2
, len
);
842 * Map 'nsgld' data descriptors from 'segment'. The function will subtract the
843 * number of bytes mapped in len.
845 static uint16_t nvme_map_sgl_data(NvmeCtrl
*n
, NvmeSg
*sg
,
846 NvmeSglDescriptor
*segment
, uint64_t nsgld
,
847 size_t *len
, NvmeCmd
*cmd
)
849 dma_addr_t addr
, trans_len
;
853 for (int i
= 0; i
< nsgld
; i
++) {
854 uint8_t type
= NVME_SGL_TYPE(segment
[i
].type
);
857 case NVME_SGL_DESCR_TYPE_DATA_BLOCK
:
859 case NVME_SGL_DESCR_TYPE_SEGMENT
:
860 case NVME_SGL_DESCR_TYPE_LAST_SEGMENT
:
861 return NVME_INVALID_NUM_SGL_DESCRS
| NVME_DNR
;
863 return NVME_SGL_DESCR_TYPE_INVALID
| NVME_DNR
;
866 dlen
= le32_to_cpu(segment
[i
].len
);
874 * All data has been mapped, but the SGL contains additional
875 * segments and/or descriptors. The controller might accept
876 * ignoring the rest of the SGL.
878 uint32_t sgls
= le32_to_cpu(n
->id_ctrl
.sgls
);
879 if (sgls
& NVME_CTRL_SGLS_EXCESS_LENGTH
) {
883 trace_pci_nvme_err_invalid_sgl_excess_length(dlen
);
884 return NVME_DATA_SGL_LEN_INVALID
| NVME_DNR
;
887 trans_len
= MIN(*len
, dlen
);
889 addr
= le64_to_cpu(segment
[i
].addr
);
891 if (UINT64_MAX
- addr
< dlen
) {
892 return NVME_DATA_SGL_LEN_INVALID
| NVME_DNR
;
895 status
= nvme_map_addr(n
, sg
, addr
, trans_len
);
906 static uint16_t nvme_map_sgl(NvmeCtrl
*n
, NvmeSg
*sg
, NvmeSglDescriptor sgl
,
907 size_t len
, NvmeCmd
*cmd
)
910 * Read the segment in chunks of 256 descriptors (one 4k page) to avoid
911 * dynamically allocating a potentially huge SGL. The spec allows the SGL
912 * to be larger (as in number of bytes required to describe the SGL
913 * descriptors and segment chain) than the command transfer size, so it is
914 * not bounded by MDTS.
916 const int SEG_CHUNK_SIZE
= 256;
918 NvmeSglDescriptor segment
[SEG_CHUNK_SIZE
], *sgld
, *last_sgld
;
926 addr
= le64_to_cpu(sgl
.addr
);
928 trace_pci_nvme_map_sgl(NVME_SGL_TYPE(sgl
.type
), len
);
930 nvme_sg_init(n
, sg
, nvme_addr_is_dma(n
, addr
));
933 * If the entire transfer can be described with a single data block it can
934 * be mapped directly.
936 if (NVME_SGL_TYPE(sgl
.type
) == NVME_SGL_DESCR_TYPE_DATA_BLOCK
) {
937 status
= nvme_map_sgl_data(n
, sg
, sgld
, 1, &len
, cmd
);
946 switch (NVME_SGL_TYPE(sgld
->type
)) {
947 case NVME_SGL_DESCR_TYPE_SEGMENT
:
948 case NVME_SGL_DESCR_TYPE_LAST_SEGMENT
:
951 return NVME_INVALID_SGL_SEG_DESCR
| NVME_DNR
;
954 seg_len
= le32_to_cpu(sgld
->len
);
956 /* check the length of the (Last) Segment descriptor */
957 if (!seg_len
|| seg_len
& 0xf) {
958 return NVME_INVALID_SGL_SEG_DESCR
| NVME_DNR
;
961 if (UINT64_MAX
- addr
< seg_len
) {
962 return NVME_DATA_SGL_LEN_INVALID
| NVME_DNR
;
965 nsgld
= seg_len
/ sizeof(NvmeSglDescriptor
);
967 while (nsgld
> SEG_CHUNK_SIZE
) {
968 if (nvme_addr_read(n
, addr
, segment
, sizeof(segment
))) {
969 trace_pci_nvme_err_addr_read(addr
);
970 status
= NVME_DATA_TRAS_ERROR
;
974 status
= nvme_map_sgl_data(n
, sg
, segment
, SEG_CHUNK_SIZE
,
980 nsgld
-= SEG_CHUNK_SIZE
;
981 addr
+= SEG_CHUNK_SIZE
* sizeof(NvmeSglDescriptor
);
984 ret
= nvme_addr_read(n
, addr
, segment
, nsgld
*
985 sizeof(NvmeSglDescriptor
));
987 trace_pci_nvme_err_addr_read(addr
);
988 status
= NVME_DATA_TRAS_ERROR
;
992 last_sgld
= &segment
[nsgld
- 1];
995 * If the segment ends with a Data Block, then we are done.
997 if (NVME_SGL_TYPE(last_sgld
->type
) == NVME_SGL_DESCR_TYPE_DATA_BLOCK
) {
998 status
= nvme_map_sgl_data(n
, sg
, segment
, nsgld
, &len
, cmd
);
1007 * If the last descriptor was not a Data Block, then the current
1008 * segment must not be a Last Segment.
1010 if (NVME_SGL_TYPE(sgld
->type
) == NVME_SGL_DESCR_TYPE_LAST_SEGMENT
) {
1011 status
= NVME_INVALID_SGL_SEG_DESCR
| NVME_DNR
;
1016 addr
= le64_to_cpu(sgld
->addr
);
1019 * Do not map the last descriptor; it will be a Segment or Last Segment
1020 * descriptor and is handled by the next iteration.
1022 status
= nvme_map_sgl_data(n
, sg
, segment
, nsgld
- 1, &len
, cmd
);
1029 /* if there is any residual left in len, the SGL was too short */
1031 status
= NVME_DATA_SGL_LEN_INVALID
| NVME_DNR
;
1035 return NVME_SUCCESS
;
1042 uint16_t nvme_map_dptr(NvmeCtrl
*n
, NvmeSg
*sg
, size_t len
,
1045 uint64_t prp1
, prp2
;
1047 switch (NVME_CMD_FLAGS_PSDT(cmd
->flags
)) {
1049 prp1
= le64_to_cpu(cmd
->dptr
.prp1
);
1050 prp2
= le64_to_cpu(cmd
->dptr
.prp2
);
1052 return nvme_map_prp(n
, sg
, prp1
, prp2
, len
);
1053 case NVME_PSDT_SGL_MPTR_CONTIGUOUS
:
1054 case NVME_PSDT_SGL_MPTR_SGL
:
1055 return nvme_map_sgl(n
, sg
, cmd
->dptr
.sgl
, len
, cmd
);
1057 return NVME_INVALID_FIELD
;
1061 static uint16_t nvme_map_mptr(NvmeCtrl
*n
, NvmeSg
*sg
, size_t len
,
1064 int psdt
= NVME_CMD_FLAGS_PSDT(cmd
->flags
);
1065 hwaddr mptr
= le64_to_cpu(cmd
->mptr
);
1068 if (psdt
== NVME_PSDT_SGL_MPTR_SGL
) {
1069 NvmeSglDescriptor sgl
;
1071 if (nvme_addr_read(n
, mptr
, &sgl
, sizeof(sgl
))) {
1072 return NVME_DATA_TRAS_ERROR
;
1075 status
= nvme_map_sgl(n
, sg
, sgl
, len
, cmd
);
1076 if (status
&& (status
& 0x7ff) == NVME_DATA_SGL_LEN_INVALID
) {
1077 status
= NVME_MD_SGL_LEN_INVALID
| NVME_DNR
;
1083 nvme_sg_init(n
, sg
, nvme_addr_is_dma(n
, mptr
));
1084 status
= nvme_map_addr(n
, sg
, mptr
, len
);
1092 static uint16_t nvme_map_data(NvmeCtrl
*n
, uint32_t nlb
, NvmeRequest
*req
)
1094 NvmeNamespace
*ns
= req
->ns
;
1095 NvmeRwCmd
*rw
= (NvmeRwCmd
*)&req
->cmd
;
1096 bool pi
= !!NVME_ID_NS_DPS_TYPE(ns
->id_ns
.dps
);
1097 bool pract
= !!(le16_to_cpu(rw
->control
) & NVME_RW_PRINFO_PRACT
);
1098 size_t len
= nvme_l2b(ns
, nlb
);
1101 if (nvme_ns_ext(ns
) &&
1102 !(pi
&& pract
&& ns
->lbaf
.ms
== nvme_pi_tuple_size(ns
))) {
1105 len
+= nvme_m2b(ns
, nlb
);
1107 status
= nvme_map_dptr(n
, &sg
, len
, &req
->cmd
);
1112 nvme_sg_init(n
, &req
->sg
, sg
.flags
& NVME_SG_DMA
);
1113 nvme_sg_split(&sg
, ns
, &req
->sg
, NULL
);
1116 return NVME_SUCCESS
;
1119 return nvme_map_dptr(n
, &req
->sg
, len
, &req
->cmd
);
1122 static uint16_t nvme_map_mdata(NvmeCtrl
*n
, uint32_t nlb
, NvmeRequest
*req
)
1124 NvmeNamespace
*ns
= req
->ns
;
1125 size_t len
= nvme_m2b(ns
, nlb
);
1128 if (nvme_ns_ext(ns
)) {
1131 len
+= nvme_l2b(ns
, nlb
);
1133 status
= nvme_map_dptr(n
, &sg
, len
, &req
->cmd
);
1138 nvme_sg_init(n
, &req
->sg
, sg
.flags
& NVME_SG_DMA
);
1139 nvme_sg_split(&sg
, ns
, NULL
, &req
->sg
);
1142 return NVME_SUCCESS
;
1145 return nvme_map_mptr(n
, &req
->sg
, len
, &req
->cmd
);
1148 static uint16_t nvme_tx_interleaved(NvmeCtrl
*n
, NvmeSg
*sg
, uint8_t *ptr
,
1149 uint32_t len
, uint32_t bytes
,
1150 int32_t skip_bytes
, int64_t offset
,
1151 NvmeTxDirection dir
)
1154 uint32_t trans_len
, count
= bytes
;
1155 bool dma
= sg
->flags
& NVME_SG_DMA
;
1160 assert(sg
->flags
& NVME_SG_ALLOC
);
1163 sge_len
= dma
? sg
->qsg
.sg
[sg_idx
].len
: sg
->iov
.iov
[sg_idx
].iov_len
;
1165 if (sge_len
- offset
< 0) {
1171 if (sge_len
== offset
) {
1177 trans_len
= MIN(len
, count
);
1178 trans_len
= MIN(trans_len
, sge_len
- offset
);
1181 addr
= sg
->qsg
.sg
[sg_idx
].base
+ offset
;
1183 addr
= (hwaddr
)(uintptr_t)sg
->iov
.iov
[sg_idx
].iov_base
+ offset
;
1186 if (dir
== NVME_TX_DIRECTION_TO_DEVICE
) {
1187 ret
= nvme_addr_read(n
, addr
, ptr
, trans_len
);
1189 ret
= nvme_addr_write(n
, addr
, ptr
, trans_len
);
1193 return NVME_DATA_TRAS_ERROR
;
1199 offset
+= trans_len
;
1203 offset
+= skip_bytes
;
1207 return NVME_SUCCESS
;
1210 static uint16_t nvme_tx(NvmeCtrl
*n
, NvmeSg
*sg
, void *ptr
, uint32_t len
,
1211 NvmeTxDirection dir
)
1213 assert(sg
->flags
& NVME_SG_ALLOC
);
1215 if (sg
->flags
& NVME_SG_DMA
) {
1216 const MemTxAttrs attrs
= MEMTXATTRS_UNSPECIFIED
;
1217 dma_addr_t residual
;
1219 if (dir
== NVME_TX_DIRECTION_TO_DEVICE
) {
1220 dma_buf_write(ptr
, len
, &residual
, &sg
->qsg
, attrs
);
1222 dma_buf_read(ptr
, len
, &residual
, &sg
->qsg
, attrs
);
1225 if (unlikely(residual
)) {
1226 trace_pci_nvme_err_invalid_dma();
1227 return NVME_INVALID_FIELD
| NVME_DNR
;
1232 if (dir
== NVME_TX_DIRECTION_TO_DEVICE
) {
1233 bytes
= qemu_iovec_to_buf(&sg
->iov
, 0, ptr
, len
);
1235 bytes
= qemu_iovec_from_buf(&sg
->iov
, 0, ptr
, len
);
1238 if (unlikely(bytes
!= len
)) {
1239 trace_pci_nvme_err_invalid_dma();
1240 return NVME_INVALID_FIELD
| NVME_DNR
;
1244 return NVME_SUCCESS
;
1247 static inline uint16_t nvme_c2h(NvmeCtrl
*n
, void *ptr
, uint32_t len
,
1252 status
= nvme_map_dptr(n
, &req
->sg
, len
, &req
->cmd
);
1257 return nvme_tx(n
, &req
->sg
, ptr
, len
, NVME_TX_DIRECTION_FROM_DEVICE
);
1260 static inline uint16_t nvme_h2c(NvmeCtrl
*n
, void *ptr
, uint32_t len
,
1265 status
= nvme_map_dptr(n
, &req
->sg
, len
, &req
->cmd
);
1270 return nvme_tx(n
, &req
->sg
, ptr
, len
, NVME_TX_DIRECTION_TO_DEVICE
);
1273 uint16_t nvme_bounce_data(NvmeCtrl
*n
, void *ptr
, uint32_t len
,
1274 NvmeTxDirection dir
, NvmeRequest
*req
)
1276 NvmeNamespace
*ns
= req
->ns
;
1277 NvmeRwCmd
*rw
= (NvmeRwCmd
*)&req
->cmd
;
1278 bool pi
= !!NVME_ID_NS_DPS_TYPE(ns
->id_ns
.dps
);
1279 bool pract
= !!(le16_to_cpu(rw
->control
) & NVME_RW_PRINFO_PRACT
);
1281 if (nvme_ns_ext(ns
) &&
1282 !(pi
&& pract
&& ns
->lbaf
.ms
== nvme_pi_tuple_size(ns
))) {
1283 return nvme_tx_interleaved(n
, &req
->sg
, ptr
, len
, ns
->lbasz
,
1284 ns
->lbaf
.ms
, 0, dir
);
1287 return nvme_tx(n
, &req
->sg
, ptr
, len
, dir
);
1290 uint16_t nvme_bounce_mdata(NvmeCtrl
*n
, void *ptr
, uint32_t len
,
1291 NvmeTxDirection dir
, NvmeRequest
*req
)
1293 NvmeNamespace
*ns
= req
->ns
;
1296 if (nvme_ns_ext(ns
)) {
1297 return nvme_tx_interleaved(n
, &req
->sg
, ptr
, len
, ns
->lbaf
.ms
,
1298 ns
->lbasz
, ns
->lbasz
, dir
);
1301 nvme_sg_unmap(&req
->sg
);
1303 status
= nvme_map_mptr(n
, &req
->sg
, len
, &req
->cmd
);
1308 return nvme_tx(n
, &req
->sg
, ptr
, len
, dir
);
1311 static inline void nvme_blk_read(BlockBackend
*blk
, int64_t offset
,
1312 BlockCompletionFunc
*cb
, NvmeRequest
*req
)
1314 assert(req
->sg
.flags
& NVME_SG_ALLOC
);
1316 if (req
->sg
.flags
& NVME_SG_DMA
) {
1317 req
->aiocb
= dma_blk_read(blk
, &req
->sg
.qsg
, offset
, BDRV_SECTOR_SIZE
,
1320 req
->aiocb
= blk_aio_preadv(blk
, offset
, &req
->sg
.iov
, 0, cb
, req
);
1324 static inline void nvme_blk_write(BlockBackend
*blk
, int64_t offset
,
1325 BlockCompletionFunc
*cb
, NvmeRequest
*req
)
1327 assert(req
->sg
.flags
& NVME_SG_ALLOC
);
1329 if (req
->sg
.flags
& NVME_SG_DMA
) {
1330 req
->aiocb
= dma_blk_write(blk
, &req
->sg
.qsg
, offset
, BDRV_SECTOR_SIZE
,
1333 req
->aiocb
= blk_aio_pwritev(blk
, offset
, &req
->sg
.iov
, 0, cb
, req
);
1337 static void nvme_update_cq_eventidx(const NvmeCQueue
*cq
)
1339 uint32_t v
= cpu_to_le32(cq
->head
);
1341 trace_pci_nvme_update_cq_eventidx(cq
->cqid
, cq
->head
);
1343 pci_dma_write(PCI_DEVICE(cq
->ctrl
), cq
->ei_addr
, &v
, sizeof(v
));
1346 static void nvme_update_cq_head(NvmeCQueue
*cq
)
1350 pci_dma_read(PCI_DEVICE(cq
->ctrl
), cq
->db_addr
, &v
, sizeof(v
));
1352 cq
->head
= le32_to_cpu(v
);
1354 trace_pci_nvme_update_cq_head(cq
->cqid
, cq
->head
);
1357 static void nvme_post_cqes(void *opaque
)
1359 NvmeCQueue
*cq
= opaque
;
1360 NvmeCtrl
*n
= cq
->ctrl
;
1361 NvmeRequest
*req
, *next
;
1362 bool pending
= cq
->head
!= cq
->tail
;
1365 QTAILQ_FOREACH_SAFE(req
, &cq
->req_list
, entry
, next
) {
1369 if (n
->dbbuf_enabled
) {
1370 nvme_update_cq_eventidx(cq
);
1371 nvme_update_cq_head(cq
);
1374 if (nvme_cq_full(cq
)) {
1379 req
->cqe
.status
= cpu_to_le16((req
->status
<< 1) | cq
->phase
);
1380 req
->cqe
.sq_id
= cpu_to_le16(sq
->sqid
);
1381 req
->cqe
.sq_head
= cpu_to_le16(sq
->head
);
1382 addr
= cq
->dma_addr
+ cq
->tail
* n
->cqe_size
;
1383 ret
= pci_dma_write(PCI_DEVICE(n
), addr
, (void *)&req
->cqe
,
1386 trace_pci_nvme_err_addr_write(addr
);
1387 trace_pci_nvme_err_cfs();
1388 stl_le_p(&n
->bar
.csts
, NVME_CSTS_FAILED
);
1391 QTAILQ_REMOVE(&cq
->req_list
, req
, entry
);
1392 nvme_inc_cq_tail(cq
);
1393 nvme_sg_unmap(&req
->sg
);
1394 QTAILQ_INSERT_TAIL(&sq
->req_list
, req
, entry
);
1396 if (cq
->tail
!= cq
->head
) {
1397 if (cq
->irq_enabled
&& !pending
) {
1401 nvme_irq_assert(n
, cq
);
1405 static void nvme_enqueue_req_completion(NvmeCQueue
*cq
, NvmeRequest
*req
)
1407 assert(cq
->cqid
== req
->sq
->cqid
);
1408 trace_pci_nvme_enqueue_req_completion(nvme_cid(req
), cq
->cqid
,
1409 le32_to_cpu(req
->cqe
.result
),
1410 le32_to_cpu(req
->cqe
.dw1
),
1414 trace_pci_nvme_err_req_status(nvme_cid(req
), nvme_nsid(req
->ns
),
1415 req
->status
, req
->cmd
.opcode
);
1418 QTAILQ_REMOVE(&req
->sq
->out_req_list
, req
, entry
);
1419 QTAILQ_INSERT_TAIL(&cq
->req_list
, req
, entry
);
1421 qemu_bh_schedule(cq
->bh
);
1424 static void nvme_process_aers(void *opaque
)
1426 NvmeCtrl
*n
= opaque
;
1427 NvmeAsyncEvent
*event
, *next
;
1429 trace_pci_nvme_process_aers(n
->aer_queued
);
1431 QTAILQ_FOREACH_SAFE(event
, &n
->aer_queue
, entry
, next
) {
1433 NvmeAerResult
*result
;
1435 /* can't post cqe if there is nothing to complete */
1436 if (!n
->outstanding_aers
) {
1437 trace_pci_nvme_no_outstanding_aers();
1441 /* ignore if masked (cqe posted, but event not cleared) */
1442 if (n
->aer_mask
& (1 << event
->result
.event_type
)) {
1443 trace_pci_nvme_aer_masked(event
->result
.event_type
, n
->aer_mask
);
1447 QTAILQ_REMOVE(&n
->aer_queue
, event
, entry
);
1450 n
->aer_mask
|= 1 << event
->result
.event_type
;
1451 n
->outstanding_aers
--;
1453 req
= n
->aer_reqs
[n
->outstanding_aers
];
1455 result
= (NvmeAerResult
*) &req
->cqe
.result
;
1456 result
->event_type
= event
->result
.event_type
;
1457 result
->event_info
= event
->result
.event_info
;
1458 result
->log_page
= event
->result
.log_page
;
1461 trace_pci_nvme_aer_post_cqe(result
->event_type
, result
->event_info
,
1464 nvme_enqueue_req_completion(&n
->admin_cq
, req
);
1468 static void nvme_enqueue_event(NvmeCtrl
*n
, uint8_t event_type
,
1469 uint8_t event_info
, uint8_t log_page
)
1471 NvmeAsyncEvent
*event
;
1473 trace_pci_nvme_enqueue_event(event_type
, event_info
, log_page
);
1475 if (n
->aer_queued
== n
->params
.aer_max_queued
) {
1476 trace_pci_nvme_enqueue_event_noqueue(n
->aer_queued
);
1480 event
= g_new(NvmeAsyncEvent
, 1);
1481 event
->result
= (NvmeAerResult
) {
1482 .event_type
= event_type
,
1483 .event_info
= event_info
,
1484 .log_page
= log_page
,
1487 QTAILQ_INSERT_TAIL(&n
->aer_queue
, event
, entry
);
1490 nvme_process_aers(n
);
1493 static void nvme_smart_event(NvmeCtrl
*n
, uint8_t event
)
1497 /* Ref SPEC <Asynchronous Event Information 0x2013 SMART / Health Status> */
1498 if (!(NVME_AEC_SMART(n
->features
.async_config
) & event
)) {
1503 case NVME_SMART_SPARE
:
1504 aer_info
= NVME_AER_INFO_SMART_SPARE_THRESH
;
1506 case NVME_SMART_TEMPERATURE
:
1507 aer_info
= NVME_AER_INFO_SMART_TEMP_THRESH
;
1509 case NVME_SMART_RELIABILITY
:
1510 case NVME_SMART_MEDIA_READ_ONLY
:
1511 case NVME_SMART_FAILED_VOLATILE_MEDIA
:
1512 case NVME_SMART_PMR_UNRELIABLE
:
1513 aer_info
= NVME_AER_INFO_SMART_RELIABILITY
;
1519 nvme_enqueue_event(n
, NVME_AER_TYPE_SMART
, aer_info
, NVME_LOG_SMART_INFO
);
1522 static void nvme_clear_events(NvmeCtrl
*n
, uint8_t event_type
)
1524 n
->aer_mask
&= ~(1 << event_type
);
1525 if (!QTAILQ_EMPTY(&n
->aer_queue
)) {
1526 nvme_process_aers(n
);
1530 static inline uint16_t nvme_check_mdts(NvmeCtrl
*n
, size_t len
)
1532 uint8_t mdts
= n
->params
.mdts
;
1534 if (mdts
&& len
> n
->page_size
<< mdts
) {
1535 trace_pci_nvme_err_mdts(len
);
1536 return NVME_INVALID_FIELD
| NVME_DNR
;
1539 return NVME_SUCCESS
;
1542 static inline uint16_t nvme_check_bounds(NvmeNamespace
*ns
, uint64_t slba
,
1545 uint64_t nsze
= le64_to_cpu(ns
->id_ns
.nsze
);
1547 if (unlikely(UINT64_MAX
- slba
< nlb
|| slba
+ nlb
> nsze
)) {
1548 trace_pci_nvme_err_invalid_lba_range(slba
, nlb
, nsze
);
1549 return NVME_LBA_RANGE
| NVME_DNR
;
1552 return NVME_SUCCESS
;
1555 static int nvme_block_status_all(NvmeNamespace
*ns
, uint64_t slba
,
1556 uint32_t nlb
, int flags
)
1558 BlockDriverState
*bs
= blk_bs(ns
->blkconf
.blk
);
1560 int64_t pnum
= 0, bytes
= nvme_l2b(ns
, nlb
);
1561 int64_t offset
= nvme_l2b(ns
, slba
);
1565 * `pnum` holds the number of bytes after offset that shares the same
1566 * allocation status as the byte at offset. If `pnum` is different from
1567 * `bytes`, we should check the allocation status of the next range and
1568 * continue this until all bytes have been checked.
1573 ret
= bdrv_block_status(bs
, offset
, bytes
, &pnum
, NULL
, NULL
);
1579 trace_pci_nvme_block_status(offset
, bytes
, pnum
, ret
,
1580 !!(ret
& BDRV_BLOCK_ZERO
));
1582 if (!(ret
& flags
)) {
1587 } while (pnum
!= bytes
);
1592 static uint16_t nvme_check_dulbe(NvmeNamespace
*ns
, uint64_t slba
,
1598 ret
= nvme_block_status_all(ns
, slba
, nlb
, BDRV_BLOCK_DATA
);
1601 error_setg_errno(&err
, -ret
, "unable to get block status");
1602 error_report_err(err
);
1604 return NVME_INTERNAL_DEV_ERROR
;
1610 return NVME_SUCCESS
;
1613 static void nvme_aio_err(NvmeRequest
*req
, int ret
)
1615 uint16_t status
= NVME_SUCCESS
;
1616 Error
*local_err
= NULL
;
1618 switch (req
->cmd
.opcode
) {
1620 status
= NVME_UNRECOVERED_READ
;
1622 case NVME_CMD_FLUSH
:
1623 case NVME_CMD_WRITE
:
1624 case NVME_CMD_WRITE_ZEROES
:
1625 case NVME_CMD_ZONE_APPEND
:
1626 status
= NVME_WRITE_FAULT
;
1629 status
= NVME_INTERNAL_DEV_ERROR
;
1633 trace_pci_nvme_err_aio(nvme_cid(req
), strerror(-ret
), status
);
1635 error_setg_errno(&local_err
, -ret
, "aio failed");
1636 error_report_err(local_err
);
1639 * Set the command status code to the first encountered error but allow a
1640 * subsequent Internal Device Error to trump it.
1642 if (req
->status
&& status
!= NVME_INTERNAL_DEV_ERROR
) {
1646 req
->status
= status
;
1649 static inline uint32_t nvme_zone_idx(NvmeNamespace
*ns
, uint64_t slba
)
1651 return ns
->zone_size_log2
> 0 ? slba
>> ns
->zone_size_log2
:
1652 slba
/ ns
->zone_size
;
1655 static inline NvmeZone
*nvme_get_zone_by_slba(NvmeNamespace
*ns
, uint64_t slba
)
1657 uint32_t zone_idx
= nvme_zone_idx(ns
, slba
);
1659 if (zone_idx
>= ns
->num_zones
) {
1663 return &ns
->zone_array
[zone_idx
];
1666 static uint16_t nvme_check_zone_state_for_write(NvmeZone
*zone
)
1668 uint64_t zslba
= zone
->d
.zslba
;
1670 switch (nvme_get_zone_state(zone
)) {
1671 case NVME_ZONE_STATE_EMPTY
:
1672 case NVME_ZONE_STATE_IMPLICITLY_OPEN
:
1673 case NVME_ZONE_STATE_EXPLICITLY_OPEN
:
1674 case NVME_ZONE_STATE_CLOSED
:
1675 return NVME_SUCCESS
;
1676 case NVME_ZONE_STATE_FULL
:
1677 trace_pci_nvme_err_zone_is_full(zslba
);
1678 return NVME_ZONE_FULL
;
1679 case NVME_ZONE_STATE_OFFLINE
:
1680 trace_pci_nvme_err_zone_is_offline(zslba
);
1681 return NVME_ZONE_OFFLINE
;
1682 case NVME_ZONE_STATE_READ_ONLY
:
1683 trace_pci_nvme_err_zone_is_read_only(zslba
);
1684 return NVME_ZONE_READ_ONLY
;
1689 return NVME_INTERNAL_DEV_ERROR
;
1692 static uint16_t nvme_check_zone_write(NvmeNamespace
*ns
, NvmeZone
*zone
,
1693 uint64_t slba
, uint32_t nlb
)
1695 uint64_t zcap
= nvme_zone_wr_boundary(zone
);
1698 status
= nvme_check_zone_state_for_write(zone
);
1703 if (zone
->d
.za
& NVME_ZA_ZRWA_VALID
) {
1704 uint64_t ezrwa
= zone
->w_ptr
+ 2 * ns
->zns
.zrwas
;
1706 if (slba
< zone
->w_ptr
|| slba
+ nlb
> ezrwa
) {
1707 trace_pci_nvme_err_zone_invalid_write(slba
, zone
->w_ptr
);
1708 return NVME_ZONE_INVALID_WRITE
;
1711 if (unlikely(slba
!= zone
->w_ptr
)) {
1712 trace_pci_nvme_err_write_not_at_wp(slba
, zone
->d
.zslba
,
1714 return NVME_ZONE_INVALID_WRITE
;
1718 if (unlikely((slba
+ nlb
) > zcap
)) {
1719 trace_pci_nvme_err_zone_boundary(slba
, nlb
, zcap
);
1720 return NVME_ZONE_BOUNDARY_ERROR
;
1723 return NVME_SUCCESS
;
1726 static uint16_t nvme_check_zone_state_for_read(NvmeZone
*zone
)
1728 switch (nvme_get_zone_state(zone
)) {
1729 case NVME_ZONE_STATE_EMPTY
:
1730 case NVME_ZONE_STATE_IMPLICITLY_OPEN
:
1731 case NVME_ZONE_STATE_EXPLICITLY_OPEN
:
1732 case NVME_ZONE_STATE_FULL
:
1733 case NVME_ZONE_STATE_CLOSED
:
1734 case NVME_ZONE_STATE_READ_ONLY
:
1735 return NVME_SUCCESS
;
1736 case NVME_ZONE_STATE_OFFLINE
:
1737 trace_pci_nvme_err_zone_is_offline(zone
->d
.zslba
);
1738 return NVME_ZONE_OFFLINE
;
1743 return NVME_INTERNAL_DEV_ERROR
;
1746 static uint16_t nvme_check_zone_read(NvmeNamespace
*ns
, uint64_t slba
,
1750 uint64_t bndry
, end
;
1753 zone
= nvme_get_zone_by_slba(ns
, slba
);
1756 bndry
= nvme_zone_rd_boundary(ns
, zone
);
1759 status
= nvme_check_zone_state_for_read(zone
);
1762 } else if (unlikely(end
> bndry
)) {
1763 if (!ns
->params
.cross_zone_read
) {
1764 status
= NVME_ZONE_BOUNDARY_ERROR
;
1767 * Read across zone boundary - check that all subsequent
1768 * zones that are being read have an appropriate state.
1772 status
= nvme_check_zone_state_for_read(zone
);
1776 } while (end
> nvme_zone_rd_boundary(ns
, zone
));
1783 static uint16_t nvme_zrm_finish(NvmeNamespace
*ns
, NvmeZone
*zone
)
1785 switch (nvme_get_zone_state(zone
)) {
1786 case NVME_ZONE_STATE_FULL
:
1787 return NVME_SUCCESS
;
1789 case NVME_ZONE_STATE_IMPLICITLY_OPEN
:
1790 case NVME_ZONE_STATE_EXPLICITLY_OPEN
:
1791 nvme_aor_dec_open(ns
);
1793 case NVME_ZONE_STATE_CLOSED
:
1794 nvme_aor_dec_active(ns
);
1796 if (zone
->d
.za
& NVME_ZA_ZRWA_VALID
) {
1797 zone
->d
.za
&= ~NVME_ZA_ZRWA_VALID
;
1798 if (ns
->params
.numzrwa
) {
1804 case NVME_ZONE_STATE_EMPTY
:
1805 nvme_assign_zone_state(ns
, zone
, NVME_ZONE_STATE_FULL
);
1806 return NVME_SUCCESS
;
1809 return NVME_ZONE_INVAL_TRANSITION
;
1813 static uint16_t nvme_zrm_close(NvmeNamespace
*ns
, NvmeZone
*zone
)
1815 switch (nvme_get_zone_state(zone
)) {
1816 case NVME_ZONE_STATE_EXPLICITLY_OPEN
:
1817 case NVME_ZONE_STATE_IMPLICITLY_OPEN
:
1818 nvme_aor_dec_open(ns
);
1819 nvme_assign_zone_state(ns
, zone
, NVME_ZONE_STATE_CLOSED
);
1821 case NVME_ZONE_STATE_CLOSED
:
1822 return NVME_SUCCESS
;
1825 return NVME_ZONE_INVAL_TRANSITION
;
1829 static uint16_t nvme_zrm_reset(NvmeNamespace
*ns
, NvmeZone
*zone
)
1831 switch (nvme_get_zone_state(zone
)) {
1832 case NVME_ZONE_STATE_EXPLICITLY_OPEN
:
1833 case NVME_ZONE_STATE_IMPLICITLY_OPEN
:
1834 nvme_aor_dec_open(ns
);
1836 case NVME_ZONE_STATE_CLOSED
:
1837 nvme_aor_dec_active(ns
);
1839 if (zone
->d
.za
& NVME_ZA_ZRWA_VALID
) {
1840 if (ns
->params
.numzrwa
) {
1846 case NVME_ZONE_STATE_FULL
:
1847 zone
->w_ptr
= zone
->d
.zslba
;
1848 zone
->d
.wp
= zone
->w_ptr
;
1849 nvme_assign_zone_state(ns
, zone
, NVME_ZONE_STATE_EMPTY
);
1851 case NVME_ZONE_STATE_EMPTY
:
1852 return NVME_SUCCESS
;
1855 return NVME_ZONE_INVAL_TRANSITION
;
1859 static void nvme_zrm_auto_transition_zone(NvmeNamespace
*ns
)
1863 if (ns
->params
.max_open_zones
&&
1864 ns
->nr_open_zones
== ns
->params
.max_open_zones
) {
1865 zone
= QTAILQ_FIRST(&ns
->imp_open_zones
);
1868 * Automatically close this implicitly open zone.
1870 QTAILQ_REMOVE(&ns
->imp_open_zones
, zone
, entry
);
1871 nvme_zrm_close(ns
, zone
);
1877 NVME_ZRM_AUTO
= 1 << 0,
1878 NVME_ZRM_ZRWA
= 1 << 1,
1881 static uint16_t nvme_zrm_open_flags(NvmeCtrl
*n
, NvmeNamespace
*ns
,
1882 NvmeZone
*zone
, int flags
)
1887 switch (nvme_get_zone_state(zone
)) {
1888 case NVME_ZONE_STATE_EMPTY
:
1893 case NVME_ZONE_STATE_CLOSED
:
1894 if (n
->params
.auto_transition_zones
) {
1895 nvme_zrm_auto_transition_zone(ns
);
1897 status
= nvme_zns_check_resources(ns
, act
, 1,
1898 (flags
& NVME_ZRM_ZRWA
) ? 1 : 0);
1904 nvme_aor_inc_active(ns
);
1907 nvme_aor_inc_open(ns
);
1909 if (flags
& NVME_ZRM_AUTO
) {
1910 nvme_assign_zone_state(ns
, zone
, NVME_ZONE_STATE_IMPLICITLY_OPEN
);
1911 return NVME_SUCCESS
;
1916 case NVME_ZONE_STATE_IMPLICITLY_OPEN
:
1917 if (flags
& NVME_ZRM_AUTO
) {
1918 return NVME_SUCCESS
;
1921 nvme_assign_zone_state(ns
, zone
, NVME_ZONE_STATE_EXPLICITLY_OPEN
);
1925 case NVME_ZONE_STATE_EXPLICITLY_OPEN
:
1926 if (flags
& NVME_ZRM_ZRWA
) {
1929 zone
->d
.za
|= NVME_ZA_ZRWA_VALID
;
1932 return NVME_SUCCESS
;
1935 return NVME_ZONE_INVAL_TRANSITION
;
1939 static inline uint16_t nvme_zrm_auto(NvmeCtrl
*n
, NvmeNamespace
*ns
,
1942 return nvme_zrm_open_flags(n
, ns
, zone
, NVME_ZRM_AUTO
);
1945 static void nvme_advance_zone_wp(NvmeNamespace
*ns
, NvmeZone
*zone
,
1950 if (zone
->d
.wp
== nvme_zone_wr_boundary(zone
)) {
1951 nvme_zrm_finish(ns
, zone
);
1955 static void nvme_zoned_zrwa_implicit_flush(NvmeNamespace
*ns
, NvmeZone
*zone
,
1958 uint16_t nzrwafgs
= DIV_ROUND_UP(nlbc
, ns
->zns
.zrwafg
);
1960 nlbc
= nzrwafgs
* ns
->zns
.zrwafg
;
1962 trace_pci_nvme_zoned_zrwa_implicit_flush(zone
->d
.zslba
, nlbc
);
1964 zone
->w_ptr
+= nlbc
;
1966 nvme_advance_zone_wp(ns
, zone
, nlbc
);
1969 static void nvme_finalize_zoned_write(NvmeNamespace
*ns
, NvmeRequest
*req
)
1971 NvmeRwCmd
*rw
= (NvmeRwCmd
*)&req
->cmd
;
1976 slba
= le64_to_cpu(rw
->slba
);
1977 nlb
= le16_to_cpu(rw
->nlb
) + 1;
1978 zone
= nvme_get_zone_by_slba(ns
, slba
);
1981 if (zone
->d
.za
& NVME_ZA_ZRWA_VALID
) {
1982 uint64_t ezrwa
= zone
->w_ptr
+ ns
->zns
.zrwas
- 1;
1983 uint64_t elba
= slba
+ nlb
- 1;
1986 nvme_zoned_zrwa_implicit_flush(ns
, zone
, elba
- ezrwa
);
1992 nvme_advance_zone_wp(ns
, zone
, nlb
);
1995 static inline bool nvme_is_write(NvmeRequest
*req
)
1997 NvmeRwCmd
*rw
= (NvmeRwCmd
*)&req
->cmd
;
1999 return rw
->opcode
== NVME_CMD_WRITE
||
2000 rw
->opcode
== NVME_CMD_ZONE_APPEND
||
2001 rw
->opcode
== NVME_CMD_WRITE_ZEROES
;
2004 static AioContext
*nvme_get_aio_context(BlockAIOCB
*acb
)
2006 return qemu_get_aio_context();
2009 static void nvme_misc_cb(void *opaque
, int ret
)
2011 NvmeRequest
*req
= opaque
;
2013 trace_pci_nvme_misc_cb(nvme_cid(req
));
2016 nvme_aio_err(req
, ret
);
2019 nvme_enqueue_req_completion(nvme_cq(req
), req
);
2022 void nvme_rw_complete_cb(void *opaque
, int ret
)
2024 NvmeRequest
*req
= opaque
;
2025 NvmeNamespace
*ns
= req
->ns
;
2026 BlockBackend
*blk
= ns
->blkconf
.blk
;
2027 BlockAcctCookie
*acct
= &req
->acct
;
2028 BlockAcctStats
*stats
= blk_get_stats(blk
);
2030 trace_pci_nvme_rw_complete_cb(nvme_cid(req
), blk_name(blk
));
2033 block_acct_failed(stats
, acct
);
2034 nvme_aio_err(req
, ret
);
2036 block_acct_done(stats
, acct
);
2039 if (ns
->params
.zoned
&& nvme_is_write(req
)) {
2040 nvme_finalize_zoned_write(ns
, req
);
2043 nvme_enqueue_req_completion(nvme_cq(req
), req
);
2046 static void nvme_rw_cb(void *opaque
, int ret
)
2048 NvmeRequest
*req
= opaque
;
2049 NvmeNamespace
*ns
= req
->ns
;
2051 BlockBackend
*blk
= ns
->blkconf
.blk
;
2053 trace_pci_nvme_rw_cb(nvme_cid(req
), blk_name(blk
));
2060 NvmeRwCmd
*rw
= (NvmeRwCmd
*)&req
->cmd
;
2061 uint64_t slba
= le64_to_cpu(rw
->slba
);
2062 uint32_t nlb
= (uint32_t)le16_to_cpu(rw
->nlb
) + 1;
2063 uint64_t offset
= nvme_moff(ns
, slba
);
2065 if (req
->cmd
.opcode
== NVME_CMD_WRITE_ZEROES
) {
2066 size_t mlen
= nvme_m2b(ns
, nlb
);
2068 req
->aiocb
= blk_aio_pwrite_zeroes(blk
, offset
, mlen
,
2070 nvme_rw_complete_cb
, req
);
2074 if (nvme_ns_ext(ns
) || req
->cmd
.mptr
) {
2077 nvme_sg_unmap(&req
->sg
);
2078 status
= nvme_map_mdata(nvme_ctrl(req
), nlb
, req
);
2084 if (req
->cmd
.opcode
== NVME_CMD_READ
) {
2085 return nvme_blk_read(blk
, offset
, nvme_rw_complete_cb
, req
);
2088 return nvme_blk_write(blk
, offset
, nvme_rw_complete_cb
, req
);
2093 nvme_rw_complete_cb(req
, ret
);
2096 static void nvme_verify_cb(void *opaque
, int ret
)
2098 NvmeBounceContext
*ctx
= opaque
;
2099 NvmeRequest
*req
= ctx
->req
;
2100 NvmeNamespace
*ns
= req
->ns
;
2101 BlockBackend
*blk
= ns
->blkconf
.blk
;
2102 BlockAcctCookie
*acct
= &req
->acct
;
2103 BlockAcctStats
*stats
= blk_get_stats(blk
);
2104 NvmeRwCmd
*rw
= (NvmeRwCmd
*)&req
->cmd
;
2105 uint64_t slba
= le64_to_cpu(rw
->slba
);
2106 uint8_t prinfo
= NVME_RW_PRINFO(le16_to_cpu(rw
->control
));
2107 uint16_t apptag
= le16_to_cpu(rw
->apptag
);
2108 uint16_t appmask
= le16_to_cpu(rw
->appmask
);
2109 uint64_t reftag
= le32_to_cpu(rw
->reftag
);
2110 uint64_t cdw3
= le32_to_cpu(rw
->cdw3
);
2113 reftag
|= cdw3
<< 32;
2115 trace_pci_nvme_verify_cb(nvme_cid(req
), prinfo
, apptag
, appmask
, reftag
);
2118 block_acct_failed(stats
, acct
);
2119 nvme_aio_err(req
, ret
);
2123 block_acct_done(stats
, acct
);
2125 if (NVME_ID_NS_DPS_TYPE(ns
->id_ns
.dps
)) {
2126 status
= nvme_dif_mangle_mdata(ns
, ctx
->mdata
.bounce
,
2127 ctx
->mdata
.iov
.size
, slba
);
2129 req
->status
= status
;
2133 req
->status
= nvme_dif_check(ns
, ctx
->data
.bounce
, ctx
->data
.iov
.size
,
2134 ctx
->mdata
.bounce
, ctx
->mdata
.iov
.size
,
2135 prinfo
, slba
, apptag
, appmask
, &reftag
);
2139 qemu_iovec_destroy(&ctx
->data
.iov
);
2140 g_free(ctx
->data
.bounce
);
2142 qemu_iovec_destroy(&ctx
->mdata
.iov
);
2143 g_free(ctx
->mdata
.bounce
);
2147 nvme_enqueue_req_completion(nvme_cq(req
), req
);
2151 static void nvme_verify_mdata_in_cb(void *opaque
, int ret
)
2153 NvmeBounceContext
*ctx
= opaque
;
2154 NvmeRequest
*req
= ctx
->req
;
2155 NvmeNamespace
*ns
= req
->ns
;
2156 NvmeRwCmd
*rw
= (NvmeRwCmd
*)&req
->cmd
;
2157 uint64_t slba
= le64_to_cpu(rw
->slba
);
2158 uint32_t nlb
= le16_to_cpu(rw
->nlb
) + 1;
2159 size_t mlen
= nvme_m2b(ns
, nlb
);
2160 uint64_t offset
= nvme_moff(ns
, slba
);
2161 BlockBackend
*blk
= ns
->blkconf
.blk
;
2163 trace_pci_nvme_verify_mdata_in_cb(nvme_cid(req
), blk_name(blk
));
2169 ctx
->mdata
.bounce
= g_malloc(mlen
);
2171 qemu_iovec_reset(&ctx
->mdata
.iov
);
2172 qemu_iovec_add(&ctx
->mdata
.iov
, ctx
->mdata
.bounce
, mlen
);
2174 req
->aiocb
= blk_aio_preadv(blk
, offset
, &ctx
->mdata
.iov
, 0,
2175 nvme_verify_cb
, ctx
);
2179 nvme_verify_cb(ctx
, ret
);
2182 struct nvme_compare_ctx
{
2194 static void nvme_compare_mdata_cb(void *opaque
, int ret
)
2196 NvmeRequest
*req
= opaque
;
2197 NvmeNamespace
*ns
= req
->ns
;
2198 NvmeCtrl
*n
= nvme_ctrl(req
);
2199 NvmeRwCmd
*rw
= (NvmeRwCmd
*)&req
->cmd
;
2200 uint8_t prinfo
= NVME_RW_PRINFO(le16_to_cpu(rw
->control
));
2201 uint16_t apptag
= le16_to_cpu(rw
->apptag
);
2202 uint16_t appmask
= le16_to_cpu(rw
->appmask
);
2203 uint64_t reftag
= le32_to_cpu(rw
->reftag
);
2204 uint64_t cdw3
= le32_to_cpu(rw
->cdw3
);
2205 struct nvme_compare_ctx
*ctx
= req
->opaque
;
2206 g_autofree
uint8_t *buf
= NULL
;
2207 BlockBackend
*blk
= ns
->blkconf
.blk
;
2208 BlockAcctCookie
*acct
= &req
->acct
;
2209 BlockAcctStats
*stats
= blk_get_stats(blk
);
2210 uint16_t status
= NVME_SUCCESS
;
2212 reftag
|= cdw3
<< 32;
2214 trace_pci_nvme_compare_mdata_cb(nvme_cid(req
));
2217 block_acct_failed(stats
, acct
);
2218 nvme_aio_err(req
, ret
);
2222 buf
= g_malloc(ctx
->mdata
.iov
.size
);
2224 status
= nvme_bounce_mdata(n
, buf
, ctx
->mdata
.iov
.size
,
2225 NVME_TX_DIRECTION_TO_DEVICE
, req
);
2227 req
->status
= status
;
2231 if (NVME_ID_NS_DPS_TYPE(ns
->id_ns
.dps
)) {
2232 uint64_t slba
= le64_to_cpu(rw
->slba
);
2234 uint8_t *mbufp
= ctx
->mdata
.bounce
;
2235 uint8_t *end
= mbufp
+ ctx
->mdata
.iov
.size
;
2238 status
= nvme_dif_check(ns
, ctx
->data
.bounce
, ctx
->data
.iov
.size
,
2239 ctx
->mdata
.bounce
, ctx
->mdata
.iov
.size
, prinfo
,
2240 slba
, apptag
, appmask
, &reftag
);
2242 req
->status
= status
;
2247 * When formatted with protection information, do not compare the DIF
2250 if (!(ns
->id_ns
.dps
& NVME_ID_NS_DPS_FIRST_EIGHT
)) {
2251 pil
= ns
->lbaf
.ms
- nvme_pi_tuple_size(ns
);
2254 for (bufp
= buf
; mbufp
< end
; bufp
+= ns
->lbaf
.ms
, mbufp
+= ns
->lbaf
.ms
) {
2255 if (memcmp(bufp
+ pil
, mbufp
+ pil
, ns
->lbaf
.ms
- pil
)) {
2256 req
->status
= NVME_CMP_FAILURE
;
2264 if (memcmp(buf
, ctx
->mdata
.bounce
, ctx
->mdata
.iov
.size
)) {
2265 req
->status
= NVME_CMP_FAILURE
;
2269 block_acct_done(stats
, acct
);
2272 qemu_iovec_destroy(&ctx
->data
.iov
);
2273 g_free(ctx
->data
.bounce
);
2275 qemu_iovec_destroy(&ctx
->mdata
.iov
);
2276 g_free(ctx
->mdata
.bounce
);
2280 nvme_enqueue_req_completion(nvme_cq(req
), req
);
2283 static void nvme_compare_data_cb(void *opaque
, int ret
)
2285 NvmeRequest
*req
= opaque
;
2286 NvmeCtrl
*n
= nvme_ctrl(req
);
2287 NvmeNamespace
*ns
= req
->ns
;
2288 BlockBackend
*blk
= ns
->blkconf
.blk
;
2289 BlockAcctCookie
*acct
= &req
->acct
;
2290 BlockAcctStats
*stats
= blk_get_stats(blk
);
2292 struct nvme_compare_ctx
*ctx
= req
->opaque
;
2293 g_autofree
uint8_t *buf
= NULL
;
2296 trace_pci_nvme_compare_data_cb(nvme_cid(req
));
2299 block_acct_failed(stats
, acct
);
2300 nvme_aio_err(req
, ret
);
2304 buf
= g_malloc(ctx
->data
.iov
.size
);
2306 status
= nvme_bounce_data(n
, buf
, ctx
->data
.iov
.size
,
2307 NVME_TX_DIRECTION_TO_DEVICE
, req
);
2309 req
->status
= status
;
2313 if (memcmp(buf
, ctx
->data
.bounce
, ctx
->data
.iov
.size
)) {
2314 req
->status
= NVME_CMP_FAILURE
;
2319 NvmeRwCmd
*rw
= (NvmeRwCmd
*)&req
->cmd
;
2320 uint64_t slba
= le64_to_cpu(rw
->slba
);
2321 uint32_t nlb
= le16_to_cpu(rw
->nlb
) + 1;
2322 size_t mlen
= nvme_m2b(ns
, nlb
);
2323 uint64_t offset
= nvme_moff(ns
, slba
);
2325 ctx
->mdata
.bounce
= g_malloc(mlen
);
2327 qemu_iovec_init(&ctx
->mdata
.iov
, 1);
2328 qemu_iovec_add(&ctx
->mdata
.iov
, ctx
->mdata
.bounce
, mlen
);
2330 req
->aiocb
= blk_aio_preadv(blk
, offset
, &ctx
->mdata
.iov
, 0,
2331 nvme_compare_mdata_cb
, req
);
2335 block_acct_done(stats
, acct
);
2338 qemu_iovec_destroy(&ctx
->data
.iov
);
2339 g_free(ctx
->data
.bounce
);
2342 nvme_enqueue_req_completion(nvme_cq(req
), req
);
2345 typedef struct NvmeDSMAIOCB
{
2351 NvmeDsmRange
*range
;
2356 static void nvme_dsm_cancel(BlockAIOCB
*aiocb
)
2358 NvmeDSMAIOCB
*iocb
= container_of(aiocb
, NvmeDSMAIOCB
, common
);
2360 /* break nvme_dsm_cb loop */
2361 iocb
->idx
= iocb
->nr
;
2362 iocb
->ret
= -ECANCELED
;
2365 blk_aio_cancel_async(iocb
->aiocb
);
2369 * We only reach this if nvme_dsm_cancel() has already been called or
2370 * the command ran to completion.
2372 assert(iocb
->idx
== iocb
->nr
);
2376 static const AIOCBInfo nvme_dsm_aiocb_info
= {
2377 .aiocb_size
= sizeof(NvmeDSMAIOCB
),
2378 .cancel_async
= nvme_dsm_cancel
,
2381 static void nvme_dsm_cb(void *opaque
, int ret
);
2383 static void nvme_dsm_md_cb(void *opaque
, int ret
)
2385 NvmeDSMAIOCB
*iocb
= opaque
;
2386 NvmeRequest
*req
= iocb
->req
;
2387 NvmeNamespace
*ns
= req
->ns
;
2388 NvmeDsmRange
*range
;
2392 if (ret
< 0 || iocb
->ret
< 0 || !ns
->lbaf
.ms
) {
2396 range
= &iocb
->range
[iocb
->idx
- 1];
2397 slba
= le64_to_cpu(range
->slba
);
2398 nlb
= le32_to_cpu(range
->nlb
);
2401 * Check that all block were discarded (zeroed); otherwise we do not zero
2405 ret
= nvme_block_status_all(ns
, slba
, nlb
, BDRV_BLOCK_ZERO
);
2411 nvme_dsm_cb(iocb
, 0);
2415 iocb
->aiocb
= blk_aio_pwrite_zeroes(ns
->blkconf
.blk
, nvme_moff(ns
, slba
),
2416 nvme_m2b(ns
, nlb
), BDRV_REQ_MAY_UNMAP
,
2421 nvme_dsm_cb(iocb
, ret
);
2424 static void nvme_dsm_cb(void *opaque
, int ret
)
2426 NvmeDSMAIOCB
*iocb
= opaque
;
2427 NvmeRequest
*req
= iocb
->req
;
2428 NvmeCtrl
*n
= nvme_ctrl(req
);
2429 NvmeNamespace
*ns
= req
->ns
;
2430 NvmeDsmRange
*range
;
2434 if (iocb
->ret
< 0) {
2436 } else if (ret
< 0) {
2442 if (iocb
->idx
== iocb
->nr
) {
2446 range
= &iocb
->range
[iocb
->idx
++];
2447 slba
= le64_to_cpu(range
->slba
);
2448 nlb
= le32_to_cpu(range
->nlb
);
2450 trace_pci_nvme_dsm_deallocate(slba
, nlb
);
2452 if (nlb
> n
->dmrsl
) {
2453 trace_pci_nvme_dsm_single_range_limit_exceeded(nlb
, n
->dmrsl
);
2457 if (nvme_check_bounds(ns
, slba
, nlb
)) {
2458 trace_pci_nvme_err_invalid_lba_range(slba
, nlb
,
2463 iocb
->aiocb
= blk_aio_pdiscard(ns
->blkconf
.blk
, nvme_l2b(ns
, slba
),
2465 nvme_dsm_md_cb
, iocb
);
2470 iocb
->common
.cb(iocb
->common
.opaque
, iocb
->ret
);
2471 qemu_aio_unref(iocb
);
2474 static uint16_t nvme_dsm(NvmeCtrl
*n
, NvmeRequest
*req
)
2476 NvmeNamespace
*ns
= req
->ns
;
2477 NvmeDsmCmd
*dsm
= (NvmeDsmCmd
*) &req
->cmd
;
2478 uint32_t attr
= le32_to_cpu(dsm
->attributes
);
2479 uint32_t nr
= (le32_to_cpu(dsm
->nr
) & 0xff) + 1;
2480 uint16_t status
= NVME_SUCCESS
;
2482 trace_pci_nvme_dsm(nr
, attr
);
2484 if (attr
& NVME_DSMGMT_AD
) {
2485 NvmeDSMAIOCB
*iocb
= blk_aio_get(&nvme_dsm_aiocb_info
, ns
->blkconf
.blk
,
2490 iocb
->range
= g_new(NvmeDsmRange
, nr
);
2494 status
= nvme_h2c(n
, (uint8_t *)iocb
->range
, sizeof(NvmeDsmRange
) * nr
,
2500 req
->aiocb
= &iocb
->common
;
2501 nvme_dsm_cb(iocb
, 0);
2503 return NVME_NO_COMPLETE
;
2509 static uint16_t nvme_verify(NvmeCtrl
*n
, NvmeRequest
*req
)
2511 NvmeRwCmd
*rw
= (NvmeRwCmd
*)&req
->cmd
;
2512 NvmeNamespace
*ns
= req
->ns
;
2513 BlockBackend
*blk
= ns
->blkconf
.blk
;
2514 uint64_t slba
= le64_to_cpu(rw
->slba
);
2515 uint32_t nlb
= le16_to_cpu(rw
->nlb
) + 1;
2516 size_t len
= nvme_l2b(ns
, nlb
);
2517 int64_t offset
= nvme_l2b(ns
, slba
);
2518 uint8_t prinfo
= NVME_RW_PRINFO(le16_to_cpu(rw
->control
));
2519 uint32_t reftag
= le32_to_cpu(rw
->reftag
);
2520 NvmeBounceContext
*ctx
= NULL
;
2523 trace_pci_nvme_verify(nvme_cid(req
), nvme_nsid(ns
), slba
, nlb
);
2525 if (NVME_ID_NS_DPS_TYPE(ns
->id_ns
.dps
)) {
2526 status
= nvme_check_prinfo(ns
, prinfo
, slba
, reftag
);
2531 if (prinfo
& NVME_PRINFO_PRACT
) {
2532 return NVME_INVALID_PROT_INFO
| NVME_DNR
;
2536 if (len
> n
->page_size
<< n
->params
.vsl
) {
2537 return NVME_INVALID_FIELD
| NVME_DNR
;
2540 status
= nvme_check_bounds(ns
, slba
, nlb
);
2545 if (NVME_ERR_REC_DULBE(ns
->features
.err_rec
)) {
2546 status
= nvme_check_dulbe(ns
, slba
, nlb
);
2552 ctx
= g_new0(NvmeBounceContext
, 1);
2555 ctx
->data
.bounce
= g_malloc(len
);
2557 qemu_iovec_init(&ctx
->data
.iov
, 1);
2558 qemu_iovec_add(&ctx
->data
.iov
, ctx
->data
.bounce
, len
);
2560 block_acct_start(blk_get_stats(blk
), &req
->acct
, ctx
->data
.iov
.size
,
2563 req
->aiocb
= blk_aio_preadv(ns
->blkconf
.blk
, offset
, &ctx
->data
.iov
, 0,
2564 nvme_verify_mdata_in_cb
, ctx
);
2565 return NVME_NO_COMPLETE
;
2568 typedef struct NvmeCopyAIOCB
{
2575 unsigned int format
;
2582 BlockAcctCookie read
;
2583 BlockAcctCookie write
;
2592 static void nvme_copy_cancel(BlockAIOCB
*aiocb
)
2594 NvmeCopyAIOCB
*iocb
= container_of(aiocb
, NvmeCopyAIOCB
, common
);
2596 iocb
->ret
= -ECANCELED
;
2599 blk_aio_cancel_async(iocb
->aiocb
);
2604 static const AIOCBInfo nvme_copy_aiocb_info
= {
2605 .aiocb_size
= sizeof(NvmeCopyAIOCB
),
2606 .cancel_async
= nvme_copy_cancel
,
2609 static void nvme_copy_done(NvmeCopyAIOCB
*iocb
)
2611 NvmeRequest
*req
= iocb
->req
;
2612 NvmeNamespace
*ns
= req
->ns
;
2613 BlockAcctStats
*stats
= blk_get_stats(ns
->blkconf
.blk
);
2615 if (iocb
->idx
!= iocb
->nr
) {
2616 req
->cqe
.result
= cpu_to_le32(iocb
->idx
);
2619 qemu_iovec_destroy(&iocb
->iov
);
2620 g_free(iocb
->bounce
);
2622 if (iocb
->ret
< 0) {
2623 block_acct_failed(stats
, &iocb
->acct
.read
);
2624 block_acct_failed(stats
, &iocb
->acct
.write
);
2626 block_acct_done(stats
, &iocb
->acct
.read
);
2627 block_acct_done(stats
, &iocb
->acct
.write
);
2630 iocb
->common
.cb(iocb
->common
.opaque
, iocb
->ret
);
2631 qemu_aio_unref(iocb
);
2634 static void nvme_do_copy(NvmeCopyAIOCB
*iocb
);
2636 static void nvme_copy_source_range_parse_format0(void *ranges
, int idx
,
2637 uint64_t *slba
, uint32_t *nlb
,
2642 NvmeCopySourceRangeFormat0
*_ranges
= ranges
;
2645 *slba
= le64_to_cpu(_ranges
[idx
].slba
);
2649 *nlb
= le16_to_cpu(_ranges
[idx
].nlb
) + 1;
2653 *apptag
= le16_to_cpu(_ranges
[idx
].apptag
);
2657 *appmask
= le16_to_cpu(_ranges
[idx
].appmask
);
2661 *reftag
= le32_to_cpu(_ranges
[idx
].reftag
);
2665 static void nvme_copy_source_range_parse_format1(void *ranges
, int idx
,
2666 uint64_t *slba
, uint32_t *nlb
,
2671 NvmeCopySourceRangeFormat1
*_ranges
= ranges
;
2674 *slba
= le64_to_cpu(_ranges
[idx
].slba
);
2678 *nlb
= le16_to_cpu(_ranges
[idx
].nlb
) + 1;
2682 *apptag
= le16_to_cpu(_ranges
[idx
].apptag
);
2686 *appmask
= le16_to_cpu(_ranges
[idx
].appmask
);
2692 *reftag
|= (uint64_t)_ranges
[idx
].sr
[4] << 40;
2693 *reftag
|= (uint64_t)_ranges
[idx
].sr
[5] << 32;
2694 *reftag
|= (uint64_t)_ranges
[idx
].sr
[6] << 24;
2695 *reftag
|= (uint64_t)_ranges
[idx
].sr
[7] << 16;
2696 *reftag
|= (uint64_t)_ranges
[idx
].sr
[8] << 8;
2697 *reftag
|= (uint64_t)_ranges
[idx
].sr
[9];
2701 static void nvme_copy_source_range_parse(void *ranges
, int idx
, uint8_t format
,
2702 uint64_t *slba
, uint32_t *nlb
,
2703 uint16_t *apptag
, uint16_t *appmask
,
2707 case NVME_COPY_FORMAT_0
:
2708 nvme_copy_source_range_parse_format0(ranges
, idx
, slba
, nlb
, apptag
,
2712 case NVME_COPY_FORMAT_1
:
2713 nvme_copy_source_range_parse_format1(ranges
, idx
, slba
, nlb
, apptag
,
2722 static void nvme_copy_out_completed_cb(void *opaque
, int ret
)
2724 NvmeCopyAIOCB
*iocb
= opaque
;
2725 NvmeRequest
*req
= iocb
->req
;
2726 NvmeNamespace
*ns
= req
->ns
;
2729 nvme_copy_source_range_parse(iocb
->ranges
, iocb
->idx
, iocb
->format
, NULL
,
2730 &nlb
, NULL
, NULL
, NULL
);
2735 } else if (iocb
->ret
< 0) {
2739 if (ns
->params
.zoned
) {
2740 nvme_advance_zone_wp(ns
, iocb
->zone
, nlb
);
2749 static void nvme_copy_out_cb(void *opaque
, int ret
)
2751 NvmeCopyAIOCB
*iocb
= opaque
;
2752 NvmeRequest
*req
= iocb
->req
;
2753 NvmeNamespace
*ns
= req
->ns
;
2758 if (ret
< 0 || iocb
->ret
< 0 || !ns
->lbaf
.ms
) {
2762 nvme_copy_source_range_parse(iocb
->ranges
, iocb
->idx
, iocb
->format
, NULL
,
2763 &nlb
, NULL
, NULL
, NULL
);
2765 mlen
= nvme_m2b(ns
, nlb
);
2766 mbounce
= iocb
->bounce
+ nvme_l2b(ns
, nlb
);
2768 qemu_iovec_reset(&iocb
->iov
);
2769 qemu_iovec_add(&iocb
->iov
, mbounce
, mlen
);
2771 iocb
->aiocb
= blk_aio_pwritev(ns
->blkconf
.blk
, nvme_moff(ns
, iocb
->slba
),
2772 &iocb
->iov
, 0, nvme_copy_out_completed_cb
,
2778 nvme_copy_out_completed_cb(iocb
, ret
);
2781 static void nvme_copy_in_completed_cb(void *opaque
, int ret
)
2783 NvmeCopyAIOCB
*iocb
= opaque
;
2784 NvmeRequest
*req
= iocb
->req
;
2785 NvmeNamespace
*ns
= req
->ns
;
2788 uint16_t apptag
, appmask
;
2796 } else if (iocb
->ret
< 0) {
2800 nvme_copy_source_range_parse(iocb
->ranges
, iocb
->idx
, iocb
->format
, &slba
,
2801 &nlb
, &apptag
, &appmask
, &reftag
);
2802 len
= nvme_l2b(ns
, nlb
);
2804 trace_pci_nvme_copy_out(iocb
->slba
, nlb
);
2806 if (NVME_ID_NS_DPS_TYPE(ns
->id_ns
.dps
)) {
2807 NvmeCopyCmd
*copy
= (NvmeCopyCmd
*)&req
->cmd
;
2809 uint16_t prinfor
= ((copy
->control
[0] >> 4) & 0xf);
2810 uint16_t prinfow
= ((copy
->control
[2] >> 2) & 0xf);
2812 size_t mlen
= nvme_m2b(ns
, nlb
);
2813 uint8_t *mbounce
= iocb
->bounce
+ nvme_l2b(ns
, nlb
);
2815 status
= nvme_dif_mangle_mdata(ns
, mbounce
, mlen
, slba
);
2819 status
= nvme_dif_check(ns
, iocb
->bounce
, len
, mbounce
, mlen
, prinfor
,
2820 slba
, apptag
, appmask
, &reftag
);
2825 apptag
= le16_to_cpu(copy
->apptag
);
2826 appmask
= le16_to_cpu(copy
->appmask
);
2828 if (prinfow
& NVME_PRINFO_PRACT
) {
2829 status
= nvme_check_prinfo(ns
, prinfow
, iocb
->slba
, iocb
->reftag
);
2834 nvme_dif_pract_generate_dif(ns
, iocb
->bounce
, len
, mbounce
, mlen
,
2835 apptag
, &iocb
->reftag
);
2837 status
= nvme_dif_check(ns
, iocb
->bounce
, len
, mbounce
, mlen
,
2838 prinfow
, iocb
->slba
, apptag
, appmask
,
2846 status
= nvme_check_bounds(ns
, iocb
->slba
, nlb
);
2851 if (ns
->params
.zoned
) {
2852 status
= nvme_check_zone_write(ns
, iocb
->zone
, iocb
->slba
, nlb
);
2857 if (!(iocb
->zone
->d
.za
& NVME_ZA_ZRWA_VALID
)) {
2858 iocb
->zone
->w_ptr
+= nlb
;
2862 qemu_iovec_reset(&iocb
->iov
);
2863 qemu_iovec_add(&iocb
->iov
, iocb
->bounce
, len
);
2865 iocb
->aiocb
= blk_aio_pwritev(ns
->blkconf
.blk
, nvme_l2b(ns
, iocb
->slba
),
2866 &iocb
->iov
, 0, nvme_copy_out_cb
, iocb
);
2871 req
->status
= status
;
2877 static void nvme_copy_in_cb(void *opaque
, int ret
)
2879 NvmeCopyAIOCB
*iocb
= opaque
;
2880 NvmeRequest
*req
= iocb
->req
;
2881 NvmeNamespace
*ns
= req
->ns
;
2885 if (ret
< 0 || iocb
->ret
< 0 || !ns
->lbaf
.ms
) {
2889 nvme_copy_source_range_parse(iocb
->ranges
, iocb
->idx
, iocb
->format
, &slba
,
2890 &nlb
, NULL
, NULL
, NULL
);
2892 qemu_iovec_reset(&iocb
->iov
);
2893 qemu_iovec_add(&iocb
->iov
, iocb
->bounce
+ nvme_l2b(ns
, nlb
),
2896 iocb
->aiocb
= blk_aio_preadv(ns
->blkconf
.blk
, nvme_moff(ns
, slba
),
2897 &iocb
->iov
, 0, nvme_copy_in_completed_cb
,
2902 nvme_copy_in_completed_cb(iocb
, ret
);
2905 static void nvme_do_copy(NvmeCopyAIOCB
*iocb
)
2907 NvmeRequest
*req
= iocb
->req
;
2908 NvmeNamespace
*ns
= req
->ns
;
2914 if (iocb
->ret
< 0) {
2918 if (iocb
->idx
== iocb
->nr
) {
2922 nvme_copy_source_range_parse(iocb
->ranges
, iocb
->idx
, iocb
->format
, &slba
,
2923 &nlb
, NULL
, NULL
, NULL
);
2924 len
= nvme_l2b(ns
, nlb
);
2926 trace_pci_nvme_copy_source_range(slba
, nlb
);
2928 if (nlb
> le16_to_cpu(ns
->id_ns
.mssrl
)) {
2929 status
= NVME_CMD_SIZE_LIMIT
| NVME_DNR
;
2933 status
= nvme_check_bounds(ns
, slba
, nlb
);
2938 if (NVME_ERR_REC_DULBE(ns
->features
.err_rec
)) {
2939 status
= nvme_check_dulbe(ns
, slba
, nlb
);
2945 if (ns
->params
.zoned
) {
2946 status
= nvme_check_zone_read(ns
, slba
, nlb
);
2952 qemu_iovec_reset(&iocb
->iov
);
2953 qemu_iovec_add(&iocb
->iov
, iocb
->bounce
, len
);
2955 iocb
->aiocb
= blk_aio_preadv(ns
->blkconf
.blk
, nvme_l2b(ns
, slba
),
2956 &iocb
->iov
, 0, nvme_copy_in_cb
, iocb
);
2960 req
->status
= status
;
2963 nvme_copy_done(iocb
);
2966 static uint16_t nvme_copy(NvmeCtrl
*n
, NvmeRequest
*req
)
2968 NvmeNamespace
*ns
= req
->ns
;
2969 NvmeCopyCmd
*copy
= (NvmeCopyCmd
*)&req
->cmd
;
2970 NvmeCopyAIOCB
*iocb
= blk_aio_get(&nvme_copy_aiocb_info
, ns
->blkconf
.blk
,
2972 uint16_t nr
= copy
->nr
+ 1;
2973 uint8_t format
= copy
->control
[0] & 0xf;
2974 uint16_t prinfor
= ((copy
->control
[0] >> 4) & 0xf);
2975 uint16_t prinfow
= ((copy
->control
[2] >> 2) & 0xf);
2976 size_t len
= sizeof(NvmeCopySourceRangeFormat0
);
2980 trace_pci_nvme_copy(nvme_cid(req
), nvme_nsid(ns
), nr
, format
);
2982 iocb
->ranges
= NULL
;
2985 if (NVME_ID_NS_DPS_TYPE(ns
->id_ns
.dps
) &&
2986 ((prinfor
& NVME_PRINFO_PRACT
) != (prinfow
& NVME_PRINFO_PRACT
))) {
2987 status
= NVME_INVALID_FIELD
| NVME_DNR
;
2991 if (!(n
->id_ctrl
.ocfs
& (1 << format
))) {
2992 trace_pci_nvme_err_copy_invalid_format(format
);
2993 status
= NVME_INVALID_FIELD
| NVME_DNR
;
2997 if (nr
> ns
->id_ns
.msrc
+ 1) {
2998 status
= NVME_CMD_SIZE_LIMIT
| NVME_DNR
;
3002 if ((ns
->pif
== 0x0 && format
!= 0x0) ||
3003 (ns
->pif
!= 0x0 && format
!= 0x1)) {
3004 status
= NVME_INVALID_FORMAT
| NVME_DNR
;
3009 len
= sizeof(NvmeCopySourceRangeFormat1
);
3012 iocb
->format
= format
;
3013 iocb
->ranges
= g_malloc_n(nr
, len
);
3014 status
= nvme_h2c(n
, (uint8_t *)iocb
->ranges
, len
* nr
, req
);
3019 iocb
->slba
= le64_to_cpu(copy
->sdlba
);
3021 if (ns
->params
.zoned
) {
3022 iocb
->zone
= nvme_get_zone_by_slba(ns
, iocb
->slba
);
3024 status
= NVME_LBA_RANGE
| NVME_DNR
;
3028 status
= nvme_zrm_auto(n
, ns
, iocb
->zone
);
3038 iocb
->reftag
= le32_to_cpu(copy
->reftag
);
3039 iocb
->reftag
|= (uint64_t)le32_to_cpu(copy
->cdw3
) << 32;
3040 iocb
->bounce
= g_malloc_n(le16_to_cpu(ns
->id_ns
.mssrl
),
3041 ns
->lbasz
+ ns
->lbaf
.ms
);
3043 qemu_iovec_init(&iocb
->iov
, 1);
3045 block_acct_start(blk_get_stats(ns
->blkconf
.blk
), &iocb
->acct
.read
, 0,
3047 block_acct_start(blk_get_stats(ns
->blkconf
.blk
), &iocb
->acct
.write
, 0,
3050 req
->aiocb
= &iocb
->common
;
3053 return NVME_NO_COMPLETE
;
3056 g_free(iocb
->ranges
);
3057 qemu_aio_unref(iocb
);
3061 static uint16_t nvme_compare(NvmeCtrl
*n
, NvmeRequest
*req
)
3063 NvmeRwCmd
*rw
= (NvmeRwCmd
*)&req
->cmd
;
3064 NvmeNamespace
*ns
= req
->ns
;
3065 BlockBackend
*blk
= ns
->blkconf
.blk
;
3066 uint64_t slba
= le64_to_cpu(rw
->slba
);
3067 uint32_t nlb
= le16_to_cpu(rw
->nlb
) + 1;
3068 uint8_t prinfo
= NVME_RW_PRINFO(le16_to_cpu(rw
->control
));
3069 size_t data_len
= nvme_l2b(ns
, nlb
);
3070 size_t len
= data_len
;
3071 int64_t offset
= nvme_l2b(ns
, slba
);
3072 struct nvme_compare_ctx
*ctx
= NULL
;
3075 trace_pci_nvme_compare(nvme_cid(req
), nvme_nsid(ns
), slba
, nlb
);
3077 if (NVME_ID_NS_DPS_TYPE(ns
->id_ns
.dps
) && (prinfo
& NVME_PRINFO_PRACT
)) {
3078 return NVME_INVALID_PROT_INFO
| NVME_DNR
;
3081 if (nvme_ns_ext(ns
)) {
3082 len
+= nvme_m2b(ns
, nlb
);
3085 status
= nvme_check_mdts(n
, len
);
3090 status
= nvme_check_bounds(ns
, slba
, nlb
);
3095 if (NVME_ERR_REC_DULBE(ns
->features
.err_rec
)) {
3096 status
= nvme_check_dulbe(ns
, slba
, nlb
);
3102 status
= nvme_map_dptr(n
, &req
->sg
, len
, &req
->cmd
);
3107 ctx
= g_new(struct nvme_compare_ctx
, 1);
3108 ctx
->data
.bounce
= g_malloc(data_len
);
3112 qemu_iovec_init(&ctx
->data
.iov
, 1);
3113 qemu_iovec_add(&ctx
->data
.iov
, ctx
->data
.bounce
, data_len
);
3115 block_acct_start(blk_get_stats(blk
), &req
->acct
, data_len
,
3117 req
->aiocb
= blk_aio_preadv(blk
, offset
, &ctx
->data
.iov
, 0,
3118 nvme_compare_data_cb
, req
);
3120 return NVME_NO_COMPLETE
;
3123 typedef struct NvmeFlushAIOCB
{
3134 static void nvme_flush_cancel(BlockAIOCB
*acb
)
3136 NvmeFlushAIOCB
*iocb
= container_of(acb
, NvmeFlushAIOCB
, common
);
3138 iocb
->ret
= -ECANCELED
;
3141 blk_aio_cancel_async(iocb
->aiocb
);
3146 static const AIOCBInfo nvme_flush_aiocb_info
= {
3147 .aiocb_size
= sizeof(NvmeFlushAIOCB
),
3148 .cancel_async
= nvme_flush_cancel
,
3149 .get_aio_context
= nvme_get_aio_context
,
3152 static void nvme_do_flush(NvmeFlushAIOCB
*iocb
);
3154 static void nvme_flush_ns_cb(void *opaque
, int ret
)
3156 NvmeFlushAIOCB
*iocb
= opaque
;
3157 NvmeNamespace
*ns
= iocb
->ns
;
3162 } else if (iocb
->ret
< 0) {
3167 trace_pci_nvme_flush_ns(iocb
->nsid
);
3170 iocb
->aiocb
= blk_aio_flush(ns
->blkconf
.blk
, nvme_flush_ns_cb
, iocb
);
3175 nvme_do_flush(iocb
);
3178 static void nvme_do_flush(NvmeFlushAIOCB
*iocb
)
3180 NvmeRequest
*req
= iocb
->req
;
3181 NvmeCtrl
*n
= nvme_ctrl(req
);
3184 if (iocb
->ret
< 0) {
3188 if (iocb
->broadcast
) {
3189 for (i
= iocb
->nsid
+ 1; i
<= NVME_MAX_NAMESPACES
; i
++) {
3190 iocb
->ns
= nvme_ns(n
, i
);
3202 nvme_flush_ns_cb(iocb
, 0);
3206 iocb
->common
.cb(iocb
->common
.opaque
, iocb
->ret
);
3207 qemu_aio_unref(iocb
);
3210 static uint16_t nvme_flush(NvmeCtrl
*n
, NvmeRequest
*req
)
3212 NvmeFlushAIOCB
*iocb
;
3213 uint32_t nsid
= le32_to_cpu(req
->cmd
.nsid
);
3216 iocb
= qemu_aio_get(&nvme_flush_aiocb_info
, NULL
, nvme_misc_cb
, req
);
3222 iocb
->broadcast
= (nsid
== NVME_NSID_BROADCAST
);
3224 if (!iocb
->broadcast
) {
3225 if (!nvme_nsid_valid(n
, nsid
)) {
3226 status
= NVME_INVALID_NSID
| NVME_DNR
;
3230 iocb
->ns
= nvme_ns(n
, nsid
);
3232 status
= NVME_INVALID_FIELD
| NVME_DNR
;
3239 req
->aiocb
= &iocb
->common
;
3240 nvme_do_flush(iocb
);
3242 return NVME_NO_COMPLETE
;
3245 qemu_aio_unref(iocb
);
3250 static uint16_t nvme_read(NvmeCtrl
*n
, NvmeRequest
*req
)
3252 NvmeRwCmd
*rw
= (NvmeRwCmd
*)&req
->cmd
;
3253 NvmeNamespace
*ns
= req
->ns
;
3254 uint64_t slba
= le64_to_cpu(rw
->slba
);
3255 uint32_t nlb
= (uint32_t)le16_to_cpu(rw
->nlb
) + 1;
3256 uint8_t prinfo
= NVME_RW_PRINFO(le16_to_cpu(rw
->control
));
3257 uint64_t data_size
= nvme_l2b(ns
, nlb
);
3258 uint64_t mapped_size
= data_size
;
3259 uint64_t data_offset
;
3260 BlockBackend
*blk
= ns
->blkconf
.blk
;
3263 if (nvme_ns_ext(ns
)) {
3264 mapped_size
+= nvme_m2b(ns
, nlb
);
3266 if (NVME_ID_NS_DPS_TYPE(ns
->id_ns
.dps
)) {
3267 bool pract
= prinfo
& NVME_PRINFO_PRACT
;
3269 if (pract
&& ns
->lbaf
.ms
== nvme_pi_tuple_size(ns
)) {
3270 mapped_size
= data_size
;
3275 trace_pci_nvme_read(nvme_cid(req
), nvme_nsid(ns
), nlb
, mapped_size
, slba
);
3277 status
= nvme_check_mdts(n
, mapped_size
);
3282 status
= nvme_check_bounds(ns
, slba
, nlb
);
3287 if (ns
->params
.zoned
) {
3288 status
= nvme_check_zone_read(ns
, slba
, nlb
);
3290 trace_pci_nvme_err_zone_read_not_ok(slba
, nlb
, status
);
3295 if (NVME_ERR_REC_DULBE(ns
->features
.err_rec
)) {
3296 status
= nvme_check_dulbe(ns
, slba
, nlb
);
3302 if (NVME_ID_NS_DPS_TYPE(ns
->id_ns
.dps
)) {
3303 return nvme_dif_rw(n
, req
);
3306 status
= nvme_map_data(n
, nlb
, req
);
3311 data_offset
= nvme_l2b(ns
, slba
);
3313 block_acct_start(blk_get_stats(blk
), &req
->acct
, data_size
,
3315 nvme_blk_read(blk
, data_offset
, nvme_rw_cb
, req
);
3316 return NVME_NO_COMPLETE
;
3319 block_acct_invalid(blk_get_stats(blk
), BLOCK_ACCT_READ
);
3320 return status
| NVME_DNR
;
3323 static uint16_t nvme_do_write(NvmeCtrl
*n
, NvmeRequest
*req
, bool append
,
3326 NvmeRwCmd
*rw
= (NvmeRwCmd
*)&req
->cmd
;
3327 NvmeNamespace
*ns
= req
->ns
;
3328 uint64_t slba
= le64_to_cpu(rw
->slba
);
3329 uint32_t nlb
= (uint32_t)le16_to_cpu(rw
->nlb
) + 1;
3330 uint16_t ctrl
= le16_to_cpu(rw
->control
);
3331 uint8_t prinfo
= NVME_RW_PRINFO(ctrl
);
3332 uint64_t data_size
= nvme_l2b(ns
, nlb
);
3333 uint64_t mapped_size
= data_size
;
3334 uint64_t data_offset
;
3336 NvmeZonedResult
*res
= (NvmeZonedResult
*)&req
->cqe
;
3337 BlockBackend
*blk
= ns
->blkconf
.blk
;
3340 if (nvme_ns_ext(ns
)) {
3341 mapped_size
+= nvme_m2b(ns
, nlb
);
3343 if (NVME_ID_NS_DPS_TYPE(ns
->id_ns
.dps
)) {
3344 bool pract
= prinfo
& NVME_PRINFO_PRACT
;
3346 if (pract
&& ns
->lbaf
.ms
== nvme_pi_tuple_size(ns
)) {
3347 mapped_size
-= nvme_m2b(ns
, nlb
);
3352 trace_pci_nvme_write(nvme_cid(req
), nvme_io_opc_str(rw
->opcode
),
3353 nvme_nsid(ns
), nlb
, mapped_size
, slba
);
3356 status
= nvme_check_mdts(n
, mapped_size
);
3362 status
= nvme_check_bounds(ns
, slba
, nlb
);
3367 if (ns
->params
.zoned
) {
3368 zone
= nvme_get_zone_by_slba(ns
, slba
);
3372 bool piremap
= !!(ctrl
& NVME_RW_PIREMAP
);
3374 if (unlikely(zone
->d
.za
& NVME_ZA_ZRWA_VALID
)) {
3375 return NVME_INVALID_ZONE_OP
| NVME_DNR
;
3378 if (unlikely(slba
!= zone
->d
.zslba
)) {
3379 trace_pci_nvme_err_append_not_at_start(slba
, zone
->d
.zslba
);
3380 status
= NVME_INVALID_FIELD
;
3384 if (n
->params
.zasl
&&
3385 data_size
> (uint64_t)n
->page_size
<< n
->params
.zasl
) {
3386 trace_pci_nvme_err_zasl(data_size
);
3387 return NVME_INVALID_FIELD
| NVME_DNR
;
3391 rw
->slba
= cpu_to_le64(slba
);
3392 res
->slba
= cpu_to_le64(slba
);
3394 switch (NVME_ID_NS_DPS_TYPE(ns
->id_ns
.dps
)) {
3395 case NVME_ID_NS_DPS_TYPE_1
:
3397 return NVME_INVALID_PROT_INFO
| NVME_DNR
;
3402 case NVME_ID_NS_DPS_TYPE_2
:
3404 uint32_t reftag
= le32_to_cpu(rw
->reftag
);
3405 rw
->reftag
= cpu_to_le32(reftag
+ (slba
- zone
->d
.zslba
));
3410 case NVME_ID_NS_DPS_TYPE_3
:
3412 return NVME_INVALID_PROT_INFO
| NVME_DNR
;
3419 status
= nvme_check_zone_write(ns
, zone
, slba
, nlb
);
3424 status
= nvme_zrm_auto(n
, ns
, zone
);
3429 if (!(zone
->d
.za
& NVME_ZA_ZRWA_VALID
)) {
3434 data_offset
= nvme_l2b(ns
, slba
);
3436 if (NVME_ID_NS_DPS_TYPE(ns
->id_ns
.dps
)) {
3437 return nvme_dif_rw(n
, req
);
3441 status
= nvme_map_data(n
, nlb
, req
);
3446 block_acct_start(blk_get_stats(blk
), &req
->acct
, data_size
,
3448 nvme_blk_write(blk
, data_offset
, nvme_rw_cb
, req
);
3450 req
->aiocb
= blk_aio_pwrite_zeroes(blk
, data_offset
, data_size
,
3451 BDRV_REQ_MAY_UNMAP
, nvme_rw_cb
,
3455 return NVME_NO_COMPLETE
;
3458 block_acct_invalid(blk_get_stats(blk
), BLOCK_ACCT_WRITE
);
3459 return status
| NVME_DNR
;
3462 static inline uint16_t nvme_write(NvmeCtrl
*n
, NvmeRequest
*req
)
3464 return nvme_do_write(n
, req
, false, false);
3467 static inline uint16_t nvme_write_zeroes(NvmeCtrl
*n
, NvmeRequest
*req
)
3469 return nvme_do_write(n
, req
, false, true);
3472 static inline uint16_t nvme_zone_append(NvmeCtrl
*n
, NvmeRequest
*req
)
3474 return nvme_do_write(n
, req
, true, false);
3477 static uint16_t nvme_get_mgmt_zone_slba_idx(NvmeNamespace
*ns
, NvmeCmd
*c
,
3478 uint64_t *slba
, uint32_t *zone_idx
)
3480 uint32_t dw10
= le32_to_cpu(c
->cdw10
);
3481 uint32_t dw11
= le32_to_cpu(c
->cdw11
);
3483 if (!ns
->params
.zoned
) {
3484 trace_pci_nvme_err_invalid_opc(c
->opcode
);
3485 return NVME_INVALID_OPCODE
| NVME_DNR
;
3488 *slba
= ((uint64_t)dw11
) << 32 | dw10
;
3489 if (unlikely(*slba
>= ns
->id_ns
.nsze
)) {
3490 trace_pci_nvme_err_invalid_lba_range(*slba
, 0, ns
->id_ns
.nsze
);
3492 return NVME_LBA_RANGE
| NVME_DNR
;
3495 *zone_idx
= nvme_zone_idx(ns
, *slba
);
3496 assert(*zone_idx
< ns
->num_zones
);
3498 return NVME_SUCCESS
;
3501 typedef uint16_t (*op_handler_t
)(NvmeNamespace
*, NvmeZone
*, NvmeZoneState
,
3504 enum NvmeZoneProcessingMask
{
3505 NVME_PROC_CURRENT_ZONE
= 0,
3506 NVME_PROC_OPENED_ZONES
= 1 << 0,
3507 NVME_PROC_CLOSED_ZONES
= 1 << 1,
3508 NVME_PROC_READ_ONLY_ZONES
= 1 << 2,
3509 NVME_PROC_FULL_ZONES
= 1 << 3,
3512 static uint16_t nvme_open_zone(NvmeNamespace
*ns
, NvmeZone
*zone
,
3513 NvmeZoneState state
, NvmeRequest
*req
)
3515 NvmeZoneSendCmd
*cmd
= (NvmeZoneSendCmd
*)&req
->cmd
;
3518 if (cmd
->zsflags
& NVME_ZSFLAG_ZRWA_ALLOC
) {
3519 uint16_t ozcs
= le16_to_cpu(ns
->id_ns_zoned
->ozcs
);
3521 if (!(ozcs
& NVME_ID_NS_ZONED_OZCS_ZRWASUP
)) {
3522 return NVME_INVALID_ZONE_OP
| NVME_DNR
;
3525 if (zone
->w_ptr
% ns
->zns
.zrwafg
) {
3526 return NVME_NOZRWA
| NVME_DNR
;
3529 flags
= NVME_ZRM_ZRWA
;
3532 return nvme_zrm_open_flags(nvme_ctrl(req
), ns
, zone
, flags
);
3535 static uint16_t nvme_close_zone(NvmeNamespace
*ns
, NvmeZone
*zone
,
3536 NvmeZoneState state
, NvmeRequest
*req
)
3538 return nvme_zrm_close(ns
, zone
);
3541 static uint16_t nvme_finish_zone(NvmeNamespace
*ns
, NvmeZone
*zone
,
3542 NvmeZoneState state
, NvmeRequest
*req
)
3544 return nvme_zrm_finish(ns
, zone
);
3547 static uint16_t nvme_offline_zone(NvmeNamespace
*ns
, NvmeZone
*zone
,
3548 NvmeZoneState state
, NvmeRequest
*req
)
3551 case NVME_ZONE_STATE_READ_ONLY
:
3552 nvme_assign_zone_state(ns
, zone
, NVME_ZONE_STATE_OFFLINE
);
3554 case NVME_ZONE_STATE_OFFLINE
:
3555 return NVME_SUCCESS
;
3557 return NVME_ZONE_INVAL_TRANSITION
;
3561 static uint16_t nvme_set_zd_ext(NvmeNamespace
*ns
, NvmeZone
*zone
)
3564 uint8_t state
= nvme_get_zone_state(zone
);
3566 if (state
== NVME_ZONE_STATE_EMPTY
) {
3567 status
= nvme_aor_check(ns
, 1, 0);
3571 nvme_aor_inc_active(ns
);
3572 zone
->d
.za
|= NVME_ZA_ZD_EXT_VALID
;
3573 nvme_assign_zone_state(ns
, zone
, NVME_ZONE_STATE_CLOSED
);
3574 return NVME_SUCCESS
;
3577 return NVME_ZONE_INVAL_TRANSITION
;
3580 static uint16_t nvme_bulk_proc_zone(NvmeNamespace
*ns
, NvmeZone
*zone
,
3581 enum NvmeZoneProcessingMask proc_mask
,
3582 op_handler_t op_hndlr
, NvmeRequest
*req
)
3584 uint16_t status
= NVME_SUCCESS
;
3585 NvmeZoneState zs
= nvme_get_zone_state(zone
);
3589 case NVME_ZONE_STATE_IMPLICITLY_OPEN
:
3590 case NVME_ZONE_STATE_EXPLICITLY_OPEN
:
3591 proc_zone
= proc_mask
& NVME_PROC_OPENED_ZONES
;
3593 case NVME_ZONE_STATE_CLOSED
:
3594 proc_zone
= proc_mask
& NVME_PROC_CLOSED_ZONES
;
3596 case NVME_ZONE_STATE_READ_ONLY
:
3597 proc_zone
= proc_mask
& NVME_PROC_READ_ONLY_ZONES
;
3599 case NVME_ZONE_STATE_FULL
:
3600 proc_zone
= proc_mask
& NVME_PROC_FULL_ZONES
;
3607 status
= op_hndlr(ns
, zone
, zs
, req
);
3613 static uint16_t nvme_do_zone_op(NvmeNamespace
*ns
, NvmeZone
*zone
,
3614 enum NvmeZoneProcessingMask proc_mask
,
3615 op_handler_t op_hndlr
, NvmeRequest
*req
)
3618 uint16_t status
= NVME_SUCCESS
;
3622 status
= op_hndlr(ns
, zone
, nvme_get_zone_state(zone
), req
);
3624 if (proc_mask
& NVME_PROC_CLOSED_ZONES
) {
3625 QTAILQ_FOREACH_SAFE(zone
, &ns
->closed_zones
, entry
, next
) {
3626 status
= nvme_bulk_proc_zone(ns
, zone
, proc_mask
, op_hndlr
,
3628 if (status
&& status
!= NVME_NO_COMPLETE
) {
3633 if (proc_mask
& NVME_PROC_OPENED_ZONES
) {
3634 QTAILQ_FOREACH_SAFE(zone
, &ns
->imp_open_zones
, entry
, next
) {
3635 status
= nvme_bulk_proc_zone(ns
, zone
, proc_mask
, op_hndlr
,
3637 if (status
&& status
!= NVME_NO_COMPLETE
) {
3642 QTAILQ_FOREACH_SAFE(zone
, &ns
->exp_open_zones
, entry
, next
) {
3643 status
= nvme_bulk_proc_zone(ns
, zone
, proc_mask
, op_hndlr
,
3645 if (status
&& status
!= NVME_NO_COMPLETE
) {
3650 if (proc_mask
& NVME_PROC_FULL_ZONES
) {
3651 QTAILQ_FOREACH_SAFE(zone
, &ns
->full_zones
, entry
, next
) {
3652 status
= nvme_bulk_proc_zone(ns
, zone
, proc_mask
, op_hndlr
,
3654 if (status
&& status
!= NVME_NO_COMPLETE
) {
3660 if (proc_mask
& NVME_PROC_READ_ONLY_ZONES
) {
3661 for (i
= 0; i
< ns
->num_zones
; i
++, zone
++) {
3662 status
= nvme_bulk_proc_zone(ns
, zone
, proc_mask
, op_hndlr
,
3664 if (status
&& status
!= NVME_NO_COMPLETE
) {
3675 typedef struct NvmeZoneResetAIOCB
{
3684 } NvmeZoneResetAIOCB
;
3686 static void nvme_zone_reset_cancel(BlockAIOCB
*aiocb
)
3688 NvmeZoneResetAIOCB
*iocb
= container_of(aiocb
, NvmeZoneResetAIOCB
, common
);
3689 NvmeRequest
*req
= iocb
->req
;
3690 NvmeNamespace
*ns
= req
->ns
;
3692 iocb
->idx
= ns
->num_zones
;
3694 iocb
->ret
= -ECANCELED
;
3697 blk_aio_cancel_async(iocb
->aiocb
);
3702 static const AIOCBInfo nvme_zone_reset_aiocb_info
= {
3703 .aiocb_size
= sizeof(NvmeZoneResetAIOCB
),
3704 .cancel_async
= nvme_zone_reset_cancel
,
3707 static void nvme_zone_reset_cb(void *opaque
, int ret
);
3709 static void nvme_zone_reset_epilogue_cb(void *opaque
, int ret
)
3711 NvmeZoneResetAIOCB
*iocb
= opaque
;
3712 NvmeRequest
*req
= iocb
->req
;
3713 NvmeNamespace
*ns
= req
->ns
;
3717 if (ret
< 0 || iocb
->ret
< 0 || !ns
->lbaf
.ms
) {
3721 moff
= nvme_moff(ns
, iocb
->zone
->d
.zslba
);
3722 count
= nvme_m2b(ns
, ns
->zone_size
);
3724 iocb
->aiocb
= blk_aio_pwrite_zeroes(ns
->blkconf
.blk
, moff
, count
,
3726 nvme_zone_reset_cb
, iocb
);
3730 nvme_zone_reset_cb(iocb
, ret
);
3733 static void nvme_zone_reset_cb(void *opaque
, int ret
)
3735 NvmeZoneResetAIOCB
*iocb
= opaque
;
3736 NvmeRequest
*req
= iocb
->req
;
3737 NvmeNamespace
*ns
= req
->ns
;
3739 if (iocb
->ret
< 0) {
3741 } else if (ret
< 0) {
3747 nvme_zrm_reset(ns
, iocb
->zone
);
3754 while (iocb
->idx
< ns
->num_zones
) {
3755 NvmeZone
*zone
= &ns
->zone_array
[iocb
->idx
++];
3757 switch (nvme_get_zone_state(zone
)) {
3758 case NVME_ZONE_STATE_EMPTY
:
3765 case NVME_ZONE_STATE_EXPLICITLY_OPEN
:
3766 case NVME_ZONE_STATE_IMPLICITLY_OPEN
:
3767 case NVME_ZONE_STATE_CLOSED
:
3768 case NVME_ZONE_STATE_FULL
:
3776 trace_pci_nvme_zns_zone_reset(zone
->d
.zslba
);
3778 iocb
->aiocb
= blk_aio_pwrite_zeroes(ns
->blkconf
.blk
,
3779 nvme_l2b(ns
, zone
->d
.zslba
),
3780 nvme_l2b(ns
, ns
->zone_size
),
3782 nvme_zone_reset_epilogue_cb
,
3790 iocb
->common
.cb(iocb
->common
.opaque
, iocb
->ret
);
3791 qemu_aio_unref(iocb
);
3794 static uint16_t nvme_zone_mgmt_send_zrwa_flush(NvmeCtrl
*n
, NvmeZone
*zone
,
3795 uint64_t elba
, NvmeRequest
*req
)
3797 NvmeNamespace
*ns
= req
->ns
;
3798 uint16_t ozcs
= le16_to_cpu(ns
->id_ns_zoned
->ozcs
);
3799 uint64_t wp
= zone
->d
.wp
;
3800 uint32_t nlb
= elba
- wp
+ 1;
3804 if (!(ozcs
& NVME_ID_NS_ZONED_OZCS_ZRWASUP
)) {
3805 return NVME_INVALID_ZONE_OP
| NVME_DNR
;
3808 if (!(zone
->d
.za
& NVME_ZA_ZRWA_VALID
)) {
3809 return NVME_INVALID_FIELD
| NVME_DNR
;
3812 if (elba
< wp
|| elba
> wp
+ ns
->zns
.zrwas
) {
3813 return NVME_ZONE_BOUNDARY_ERROR
| NVME_DNR
;
3816 if (nlb
% ns
->zns
.zrwafg
) {
3817 return NVME_INVALID_FIELD
| NVME_DNR
;
3820 status
= nvme_zrm_auto(n
, ns
, zone
);
3827 nvme_advance_zone_wp(ns
, zone
, nlb
);
3829 return NVME_SUCCESS
;
3832 static uint16_t nvme_zone_mgmt_send(NvmeCtrl
*n
, NvmeRequest
*req
)
3834 NvmeZoneSendCmd
*cmd
= (NvmeZoneSendCmd
*)&req
->cmd
;
3835 NvmeNamespace
*ns
= req
->ns
;
3837 NvmeZoneResetAIOCB
*iocb
;
3840 uint32_t zone_idx
= 0;
3842 uint8_t action
= cmd
->zsa
;
3844 enum NvmeZoneProcessingMask proc_mask
= NVME_PROC_CURRENT_ZONE
;
3846 all
= cmd
->zsflags
& NVME_ZSFLAG_SELECT_ALL
;
3848 req
->status
= NVME_SUCCESS
;
3851 status
= nvme_get_mgmt_zone_slba_idx(ns
, &req
->cmd
, &slba
, &zone_idx
);
3857 zone
= &ns
->zone_array
[zone_idx
];
3858 if (slba
!= zone
->d
.zslba
&& action
!= NVME_ZONE_ACTION_ZRWA_FLUSH
) {
3859 trace_pci_nvme_err_unaligned_zone_cmd(action
, slba
, zone
->d
.zslba
);
3860 return NVME_INVALID_FIELD
| NVME_DNR
;
3865 case NVME_ZONE_ACTION_OPEN
:
3867 proc_mask
= NVME_PROC_CLOSED_ZONES
;
3869 trace_pci_nvme_open_zone(slba
, zone_idx
, all
);
3870 status
= nvme_do_zone_op(ns
, zone
, proc_mask
, nvme_open_zone
, req
);
3873 case NVME_ZONE_ACTION_CLOSE
:
3875 proc_mask
= NVME_PROC_OPENED_ZONES
;
3877 trace_pci_nvme_close_zone(slba
, zone_idx
, all
);
3878 status
= nvme_do_zone_op(ns
, zone
, proc_mask
, nvme_close_zone
, req
);
3881 case NVME_ZONE_ACTION_FINISH
:
3883 proc_mask
= NVME_PROC_OPENED_ZONES
| NVME_PROC_CLOSED_ZONES
;
3885 trace_pci_nvme_finish_zone(slba
, zone_idx
, all
);
3886 status
= nvme_do_zone_op(ns
, zone
, proc_mask
, nvme_finish_zone
, req
);
3889 case NVME_ZONE_ACTION_RESET
:
3890 trace_pci_nvme_reset_zone(slba
, zone_idx
, all
);
3892 iocb
= blk_aio_get(&nvme_zone_reset_aiocb_info
, ns
->blkconf
.blk
,
3898 iocb
->idx
= zone_idx
;
3901 req
->aiocb
= &iocb
->common
;
3902 nvme_zone_reset_cb(iocb
, 0);
3904 return NVME_NO_COMPLETE
;
3906 case NVME_ZONE_ACTION_OFFLINE
:
3908 proc_mask
= NVME_PROC_READ_ONLY_ZONES
;
3910 trace_pci_nvme_offline_zone(slba
, zone_idx
, all
);
3911 status
= nvme_do_zone_op(ns
, zone
, proc_mask
, nvme_offline_zone
, req
);
3914 case NVME_ZONE_ACTION_SET_ZD_EXT
:
3915 trace_pci_nvme_set_descriptor_extension(slba
, zone_idx
);
3916 if (all
|| !ns
->params
.zd_extension_size
) {
3917 return NVME_INVALID_FIELD
| NVME_DNR
;
3919 zd_ext
= nvme_get_zd_extension(ns
, zone_idx
);
3920 status
= nvme_h2c(n
, zd_ext
, ns
->params
.zd_extension_size
, req
);
3922 trace_pci_nvme_err_zd_extension_map_error(zone_idx
);
3926 status
= nvme_set_zd_ext(ns
, zone
);
3927 if (status
== NVME_SUCCESS
) {
3928 trace_pci_nvme_zd_extension_set(zone_idx
);
3933 case NVME_ZONE_ACTION_ZRWA_FLUSH
:
3935 return NVME_INVALID_FIELD
| NVME_DNR
;
3938 return nvme_zone_mgmt_send_zrwa_flush(n
, zone
, slba
, req
);
3941 trace_pci_nvme_err_invalid_mgmt_action(action
);
3942 status
= NVME_INVALID_FIELD
;
3945 if (status
== NVME_ZONE_INVAL_TRANSITION
) {
3946 trace_pci_nvme_err_invalid_zone_state_transition(action
, slba
,
3956 static bool nvme_zone_matches_filter(uint32_t zafs
, NvmeZone
*zl
)
3958 NvmeZoneState zs
= nvme_get_zone_state(zl
);
3961 case NVME_ZONE_REPORT_ALL
:
3963 case NVME_ZONE_REPORT_EMPTY
:
3964 return zs
== NVME_ZONE_STATE_EMPTY
;
3965 case NVME_ZONE_REPORT_IMPLICITLY_OPEN
:
3966 return zs
== NVME_ZONE_STATE_IMPLICITLY_OPEN
;
3967 case NVME_ZONE_REPORT_EXPLICITLY_OPEN
:
3968 return zs
== NVME_ZONE_STATE_EXPLICITLY_OPEN
;
3969 case NVME_ZONE_REPORT_CLOSED
:
3970 return zs
== NVME_ZONE_STATE_CLOSED
;
3971 case NVME_ZONE_REPORT_FULL
:
3972 return zs
== NVME_ZONE_STATE_FULL
;
3973 case NVME_ZONE_REPORT_READ_ONLY
:
3974 return zs
== NVME_ZONE_STATE_READ_ONLY
;
3975 case NVME_ZONE_REPORT_OFFLINE
:
3976 return zs
== NVME_ZONE_STATE_OFFLINE
;
3982 static uint16_t nvme_zone_mgmt_recv(NvmeCtrl
*n
, NvmeRequest
*req
)
3984 NvmeCmd
*cmd
= (NvmeCmd
*)&req
->cmd
;
3985 NvmeNamespace
*ns
= req
->ns
;
3986 /* cdw12 is zero-based number of dwords to return. Convert to bytes */
3987 uint32_t data_size
= (le32_to_cpu(cmd
->cdw12
) + 1) << 2;
3988 uint32_t dw13
= le32_to_cpu(cmd
->cdw13
);
3989 uint32_t zone_idx
, zra
, zrasf
, partial
;
3990 uint64_t max_zones
, nr_zones
= 0;
3995 NvmeZoneReportHeader
*header
;
3997 size_t zone_entry_sz
;
4000 req
->status
= NVME_SUCCESS
;
4002 status
= nvme_get_mgmt_zone_slba_idx(ns
, cmd
, &slba
, &zone_idx
);
4008 if (zra
!= NVME_ZONE_REPORT
&& zra
!= NVME_ZONE_REPORT_EXTENDED
) {
4009 return NVME_INVALID_FIELD
| NVME_DNR
;
4011 if (zra
== NVME_ZONE_REPORT_EXTENDED
&& !ns
->params
.zd_extension_size
) {
4012 return NVME_INVALID_FIELD
| NVME_DNR
;
4015 zrasf
= (dw13
>> 8) & 0xff;
4016 if (zrasf
> NVME_ZONE_REPORT_OFFLINE
) {
4017 return NVME_INVALID_FIELD
| NVME_DNR
;
4020 if (data_size
< sizeof(NvmeZoneReportHeader
)) {
4021 return NVME_INVALID_FIELD
| NVME_DNR
;
4024 status
= nvme_check_mdts(n
, data_size
);
4029 partial
= (dw13
>> 16) & 0x01;
4031 zone_entry_sz
= sizeof(NvmeZoneDescr
);
4032 if (zra
== NVME_ZONE_REPORT_EXTENDED
) {
4033 zone_entry_sz
+= ns
->params
.zd_extension_size
;
4036 max_zones
= (data_size
- sizeof(NvmeZoneReportHeader
)) / zone_entry_sz
;
4037 buf
= g_malloc0(data_size
);
4039 zone
= &ns
->zone_array
[zone_idx
];
4040 for (i
= zone_idx
; i
< ns
->num_zones
; i
++) {
4041 if (partial
&& nr_zones
>= max_zones
) {
4044 if (nvme_zone_matches_filter(zrasf
, zone
++)) {
4049 header
->nr_zones
= cpu_to_le64(nr_zones
);
4051 buf_p
= buf
+ sizeof(NvmeZoneReportHeader
);
4052 for (; zone_idx
< ns
->num_zones
&& max_zones
> 0; zone_idx
++) {
4053 zone
= &ns
->zone_array
[zone_idx
];
4054 if (nvme_zone_matches_filter(zrasf
, zone
)) {
4056 buf_p
+= sizeof(NvmeZoneDescr
);
4060 z
->zcap
= cpu_to_le64(zone
->d
.zcap
);
4061 z
->zslba
= cpu_to_le64(zone
->d
.zslba
);
4064 if (nvme_wp_is_valid(zone
)) {
4065 z
->wp
= cpu_to_le64(zone
->d
.wp
);
4067 z
->wp
= cpu_to_le64(~0ULL);
4070 if (zra
== NVME_ZONE_REPORT_EXTENDED
) {
4071 if (zone
->d
.za
& NVME_ZA_ZD_EXT_VALID
) {
4072 memcpy(buf_p
, nvme_get_zd_extension(ns
, zone_idx
),
4073 ns
->params
.zd_extension_size
);
4075 buf_p
+= ns
->params
.zd_extension_size
;
4082 status
= nvme_c2h(n
, (uint8_t *)buf
, data_size
, req
);
4089 static uint16_t nvme_io_cmd(NvmeCtrl
*n
, NvmeRequest
*req
)
4092 uint32_t nsid
= le32_to_cpu(req
->cmd
.nsid
);
4094 trace_pci_nvme_io_cmd(nvme_cid(req
), nsid
, nvme_sqid(req
),
4095 req
->cmd
.opcode
, nvme_io_opc_str(req
->cmd
.opcode
));
4097 if (!nvme_nsid_valid(n
, nsid
)) {
4098 return NVME_INVALID_NSID
| NVME_DNR
;
4102 * In the base NVM command set, Flush may apply to all namespaces
4103 * (indicated by NSID being set to FFFFFFFFh). But if that feature is used
4104 * along with TP 4056 (Namespace Types), it may be pretty screwed up.
4106 * If NSID is indeed set to FFFFFFFFh, we simply cannot associate the
4107 * opcode with a specific command since we cannot determine a unique I/O
4108 * command set. Opcode 0h could have any other meaning than something
4109 * equivalent to flushing and say it DOES have completely different
4110 * semantics in some other command set - does an NSID of FFFFFFFFh then
4111 * mean "for all namespaces, apply whatever command set specific command
4112 * that uses the 0h opcode?" Or does it mean "for all namespaces, apply
4113 * whatever command that uses the 0h opcode if, and only if, it allows NSID
4116 * Anyway (and luckily), for now, we do not care about this since the
4117 * device only supports namespace types that includes the NVM Flush command
4118 * (NVM and Zoned), so always do an NVM Flush.
4120 if (req
->cmd
.opcode
== NVME_CMD_FLUSH
) {
4121 return nvme_flush(n
, req
);
4124 ns
= nvme_ns(n
, nsid
);
4125 if (unlikely(!ns
)) {
4126 return NVME_INVALID_FIELD
| NVME_DNR
;
4129 if (!(ns
->iocs
[req
->cmd
.opcode
] & NVME_CMD_EFF_CSUPP
)) {
4130 trace_pci_nvme_err_invalid_opc(req
->cmd
.opcode
);
4131 return NVME_INVALID_OPCODE
| NVME_DNR
;
4138 if (NVME_CMD_FLAGS_FUSE(req
->cmd
.flags
)) {
4139 return NVME_INVALID_FIELD
;
4144 switch (req
->cmd
.opcode
) {
4145 case NVME_CMD_WRITE_ZEROES
:
4146 return nvme_write_zeroes(n
, req
);
4147 case NVME_CMD_ZONE_APPEND
:
4148 return nvme_zone_append(n
, req
);
4149 case NVME_CMD_WRITE
:
4150 return nvme_write(n
, req
);
4152 return nvme_read(n
, req
);
4153 case NVME_CMD_COMPARE
:
4154 return nvme_compare(n
, req
);
4156 return nvme_dsm(n
, req
);
4157 case NVME_CMD_VERIFY
:
4158 return nvme_verify(n
, req
);
4160 return nvme_copy(n
, req
);
4161 case NVME_CMD_ZONE_MGMT_SEND
:
4162 return nvme_zone_mgmt_send(n
, req
);
4163 case NVME_CMD_ZONE_MGMT_RECV
:
4164 return nvme_zone_mgmt_recv(n
, req
);
4169 return NVME_INVALID_OPCODE
| NVME_DNR
;
4172 static void nvme_cq_notifier(EventNotifier
*e
)
4174 NvmeCQueue
*cq
= container_of(e
, NvmeCQueue
, notifier
);
4175 NvmeCtrl
*n
= cq
->ctrl
;
4177 if (!event_notifier_test_and_clear(e
)) {
4181 nvme_update_cq_head(cq
);
4183 if (cq
->tail
== cq
->head
) {
4184 if (cq
->irq_enabled
) {
4188 nvme_irq_deassert(n
, cq
);
4191 qemu_bh_schedule(cq
->bh
);
4194 static int nvme_init_cq_ioeventfd(NvmeCQueue
*cq
)
4196 NvmeCtrl
*n
= cq
->ctrl
;
4197 uint16_t offset
= (cq
->cqid
<< 3) + (1 << 2);
4200 ret
= event_notifier_init(&cq
->notifier
, 0);
4205 event_notifier_set_handler(&cq
->notifier
, nvme_cq_notifier
);
4206 memory_region_add_eventfd(&n
->iomem
,
4207 0x1000 + offset
, 4, false, 0, &cq
->notifier
);
4212 static void nvme_sq_notifier(EventNotifier
*e
)
4214 NvmeSQueue
*sq
= container_of(e
, NvmeSQueue
, notifier
);
4216 if (!event_notifier_test_and_clear(e
)) {
4220 nvme_process_sq(sq
);
4223 static int nvme_init_sq_ioeventfd(NvmeSQueue
*sq
)
4225 NvmeCtrl
*n
= sq
->ctrl
;
4226 uint16_t offset
= sq
->sqid
<< 3;
4229 ret
= event_notifier_init(&sq
->notifier
, 0);
4234 event_notifier_set_handler(&sq
->notifier
, nvme_sq_notifier
);
4235 memory_region_add_eventfd(&n
->iomem
,
4236 0x1000 + offset
, 4, false, 0, &sq
->notifier
);
4241 static void nvme_free_sq(NvmeSQueue
*sq
, NvmeCtrl
*n
)
4243 uint16_t offset
= sq
->sqid
<< 3;
4245 n
->sq
[sq
->sqid
] = NULL
;
4246 qemu_bh_delete(sq
->bh
);
4247 if (sq
->ioeventfd_enabled
) {
4248 memory_region_del_eventfd(&n
->iomem
,
4249 0x1000 + offset
, 4, false, 0, &sq
->notifier
);
4250 event_notifier_set_handler(&sq
->notifier
, NULL
);
4251 event_notifier_cleanup(&sq
->notifier
);
4259 static uint16_t nvme_del_sq(NvmeCtrl
*n
, NvmeRequest
*req
)
4261 NvmeDeleteQ
*c
= (NvmeDeleteQ
*)&req
->cmd
;
4262 NvmeRequest
*r
, *next
;
4265 uint16_t qid
= le16_to_cpu(c
->qid
);
4267 if (unlikely(!qid
|| nvme_check_sqid(n
, qid
))) {
4268 trace_pci_nvme_err_invalid_del_sq(qid
);
4269 return NVME_INVALID_QID
| NVME_DNR
;
4272 trace_pci_nvme_del_sq(qid
);
4275 while (!QTAILQ_EMPTY(&sq
->out_req_list
)) {
4276 r
= QTAILQ_FIRST(&sq
->out_req_list
);
4278 blk_aio_cancel(r
->aiocb
);
4281 assert(QTAILQ_EMPTY(&sq
->out_req_list
));
4283 if (!nvme_check_cqid(n
, sq
->cqid
)) {
4284 cq
= n
->cq
[sq
->cqid
];
4285 QTAILQ_REMOVE(&cq
->sq_list
, sq
, entry
);
4288 QTAILQ_FOREACH_SAFE(r
, &cq
->req_list
, entry
, next
) {
4290 QTAILQ_REMOVE(&cq
->req_list
, r
, entry
);
4291 QTAILQ_INSERT_TAIL(&sq
->req_list
, r
, entry
);
4296 nvme_free_sq(sq
, n
);
4297 return NVME_SUCCESS
;
4300 static void nvme_init_sq(NvmeSQueue
*sq
, NvmeCtrl
*n
, uint64_t dma_addr
,
4301 uint16_t sqid
, uint16_t cqid
, uint16_t size
)
4307 sq
->dma_addr
= dma_addr
;
4311 sq
->head
= sq
->tail
= 0;
4312 sq
->io_req
= g_new0(NvmeRequest
, sq
->size
);
4314 QTAILQ_INIT(&sq
->req_list
);
4315 QTAILQ_INIT(&sq
->out_req_list
);
4316 for (i
= 0; i
< sq
->size
; i
++) {
4317 sq
->io_req
[i
].sq
= sq
;
4318 QTAILQ_INSERT_TAIL(&(sq
->req_list
), &sq
->io_req
[i
], entry
);
4321 sq
->bh
= qemu_bh_new(nvme_process_sq
, sq
);
4323 if (n
->dbbuf_enabled
) {
4324 sq
->db_addr
= n
->dbbuf_dbs
+ (sqid
<< 3);
4325 sq
->ei_addr
= n
->dbbuf_eis
+ (sqid
<< 3);
4327 if (n
->params
.ioeventfd
&& sq
->sqid
!= 0) {
4328 if (!nvme_init_sq_ioeventfd(sq
)) {
4329 sq
->ioeventfd_enabled
= true;
4334 assert(n
->cq
[cqid
]);
4336 QTAILQ_INSERT_TAIL(&(cq
->sq_list
), sq
, entry
);
4340 static uint16_t nvme_create_sq(NvmeCtrl
*n
, NvmeRequest
*req
)
4343 NvmeCreateSq
*c
= (NvmeCreateSq
*)&req
->cmd
;
4345 uint16_t cqid
= le16_to_cpu(c
->cqid
);
4346 uint16_t sqid
= le16_to_cpu(c
->sqid
);
4347 uint16_t qsize
= le16_to_cpu(c
->qsize
);
4348 uint16_t qflags
= le16_to_cpu(c
->sq_flags
);
4349 uint64_t prp1
= le64_to_cpu(c
->prp1
);
4351 trace_pci_nvme_create_sq(prp1
, sqid
, cqid
, qsize
, qflags
);
4353 if (unlikely(!cqid
|| nvme_check_cqid(n
, cqid
))) {
4354 trace_pci_nvme_err_invalid_create_sq_cqid(cqid
);
4355 return NVME_INVALID_CQID
| NVME_DNR
;
4357 if (unlikely(!sqid
|| sqid
> n
->conf_ioqpairs
|| n
->sq
[sqid
] != NULL
)) {
4358 trace_pci_nvme_err_invalid_create_sq_sqid(sqid
);
4359 return NVME_INVALID_QID
| NVME_DNR
;
4361 if (unlikely(!qsize
|| qsize
> NVME_CAP_MQES(ldq_le_p(&n
->bar
.cap
)))) {
4362 trace_pci_nvme_err_invalid_create_sq_size(qsize
);
4363 return NVME_MAX_QSIZE_EXCEEDED
| NVME_DNR
;
4365 if (unlikely(prp1
& (n
->page_size
- 1))) {
4366 trace_pci_nvme_err_invalid_create_sq_addr(prp1
);
4367 return NVME_INVALID_PRP_OFFSET
| NVME_DNR
;
4369 if (unlikely(!(NVME_SQ_FLAGS_PC(qflags
)))) {
4370 trace_pci_nvme_err_invalid_create_sq_qflags(NVME_SQ_FLAGS_PC(qflags
));
4371 return NVME_INVALID_FIELD
| NVME_DNR
;
4373 sq
= g_malloc0(sizeof(*sq
));
4374 nvme_init_sq(sq
, n
, prp1
, sqid
, cqid
, qsize
+ 1);
4375 return NVME_SUCCESS
;
4379 uint64_t units_read
;
4380 uint64_t units_written
;
4381 uint64_t read_commands
;
4382 uint64_t write_commands
;
4385 static void nvme_set_blk_stats(NvmeNamespace
*ns
, struct nvme_stats
*stats
)
4387 BlockAcctStats
*s
= blk_get_stats(ns
->blkconf
.blk
);
4389 stats
->units_read
+= s
->nr_bytes
[BLOCK_ACCT_READ
];
4390 stats
->units_written
+= s
->nr_bytes
[BLOCK_ACCT_WRITE
];
4391 stats
->read_commands
+= s
->nr_ops
[BLOCK_ACCT_READ
];
4392 stats
->write_commands
+= s
->nr_ops
[BLOCK_ACCT_WRITE
];
4395 static uint16_t nvme_smart_info(NvmeCtrl
*n
, uint8_t rae
, uint32_t buf_len
,
4396 uint64_t off
, NvmeRequest
*req
)
4398 uint32_t nsid
= le32_to_cpu(req
->cmd
.nsid
);
4399 struct nvme_stats stats
= { 0 };
4400 NvmeSmartLog smart
= { 0 };
4404 uint64_t u_read
, u_written
;
4406 if (off
>= sizeof(smart
)) {
4407 return NVME_INVALID_FIELD
| NVME_DNR
;
4410 if (nsid
!= 0xffffffff) {
4411 ns
= nvme_ns(n
, nsid
);
4413 return NVME_INVALID_NSID
| NVME_DNR
;
4415 nvme_set_blk_stats(ns
, &stats
);
4419 for (i
= 1; i
<= NVME_MAX_NAMESPACES
; i
++) {
4424 nvme_set_blk_stats(ns
, &stats
);
4428 trans_len
= MIN(sizeof(smart
) - off
, buf_len
);
4429 smart
.critical_warning
= n
->smart_critical_warning
;
4431 u_read
= DIV_ROUND_UP(stats
.units_read
>> BDRV_SECTOR_BITS
, 1000);
4432 u_written
= DIV_ROUND_UP(stats
.units_written
>> BDRV_SECTOR_BITS
, 1000);
4434 smart
.data_units_read
[0] = cpu_to_le64(u_read
);
4435 smart
.data_units_written
[0] = cpu_to_le64(u_written
);
4436 smart
.host_read_commands
[0] = cpu_to_le64(stats
.read_commands
);
4437 smart
.host_write_commands
[0] = cpu_to_le64(stats
.write_commands
);
4439 smart
.temperature
= cpu_to_le16(n
->temperature
);
4441 if ((n
->temperature
>= n
->features
.temp_thresh_hi
) ||
4442 (n
->temperature
<= n
->features
.temp_thresh_low
)) {
4443 smart
.critical_warning
|= NVME_SMART_TEMPERATURE
;
4446 current_ms
= qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
);
4447 smart
.power_on_hours
[0] =
4448 cpu_to_le64((((current_ms
- n
->starttime_ms
) / 1000) / 60) / 60);
4451 nvme_clear_events(n
, NVME_AER_TYPE_SMART
);
4454 return nvme_c2h(n
, (uint8_t *) &smart
+ off
, trans_len
, req
);
4457 static uint16_t nvme_endgrp_info(NvmeCtrl
*n
, uint8_t rae
, uint32_t buf_len
,
4458 uint64_t off
, NvmeRequest
*req
)
4460 uint32_t dw11
= le32_to_cpu(req
->cmd
.cdw11
);
4461 uint16_t endgrpid
= (dw11
>> 16) & 0xffff;
4462 struct nvme_stats stats
= {};
4463 NvmeEndGrpLog info
= {};
4466 if (!n
->subsys
|| endgrpid
!= 0x1) {
4467 return NVME_INVALID_FIELD
| NVME_DNR
;
4470 if (off
>= sizeof(info
)) {
4471 return NVME_INVALID_FIELD
| NVME_DNR
;
4474 for (i
= 1; i
<= NVME_MAX_NAMESPACES
; i
++) {
4475 NvmeNamespace
*ns
= nvme_subsys_ns(n
->subsys
, i
);
4480 nvme_set_blk_stats(ns
, &stats
);
4483 info
.data_units_read
[0] =
4484 cpu_to_le64(DIV_ROUND_UP(stats
.units_read
/ 1000000000, 1000000000));
4485 info
.data_units_written
[0] =
4486 cpu_to_le64(DIV_ROUND_UP(stats
.units_written
/ 1000000000, 1000000000));
4487 info
.media_units_written
[0] =
4488 cpu_to_le64(DIV_ROUND_UP(stats
.units_written
/ 1000000000, 1000000000));
4490 info
.host_read_commands
[0] = cpu_to_le64(stats
.read_commands
);
4491 info
.host_write_commands
[0] = cpu_to_le64(stats
.write_commands
);
4493 buf_len
= MIN(sizeof(info
) - off
, buf_len
);
4495 return nvme_c2h(n
, (uint8_t *)&info
+ off
, buf_len
, req
);
4499 static uint16_t nvme_fw_log_info(NvmeCtrl
*n
, uint32_t buf_len
, uint64_t off
,
4503 NvmeFwSlotInfoLog fw_log
= {
4507 if (off
>= sizeof(fw_log
)) {
4508 return NVME_INVALID_FIELD
| NVME_DNR
;
4511 strpadcpy((char *)&fw_log
.frs1
, sizeof(fw_log
.frs1
), "1.0", ' ');
4512 trans_len
= MIN(sizeof(fw_log
) - off
, buf_len
);
4514 return nvme_c2h(n
, (uint8_t *) &fw_log
+ off
, trans_len
, req
);
4517 static uint16_t nvme_error_info(NvmeCtrl
*n
, uint8_t rae
, uint32_t buf_len
,
4518 uint64_t off
, NvmeRequest
*req
)
4521 NvmeErrorLog errlog
;
4523 if (off
>= sizeof(errlog
)) {
4524 return NVME_INVALID_FIELD
| NVME_DNR
;
4528 nvme_clear_events(n
, NVME_AER_TYPE_ERROR
);
4531 memset(&errlog
, 0x0, sizeof(errlog
));
4532 trans_len
= MIN(sizeof(errlog
) - off
, buf_len
);
4534 return nvme_c2h(n
, (uint8_t *)&errlog
, trans_len
, req
);
4537 static uint16_t nvme_changed_nslist(NvmeCtrl
*n
, uint8_t rae
, uint32_t buf_len
,
4538 uint64_t off
, NvmeRequest
*req
)
4540 uint32_t nslist
[1024];
4545 if (off
>= sizeof(nslist
)) {
4546 trace_pci_nvme_err_invalid_log_page_offset(off
, sizeof(nslist
));
4547 return NVME_INVALID_FIELD
| NVME_DNR
;
4550 memset(nslist
, 0x0, sizeof(nslist
));
4551 trans_len
= MIN(sizeof(nslist
) - off
, buf_len
);
4553 while ((nsid
= find_first_bit(n
->changed_nsids
, NVME_CHANGED_NSID_SIZE
)) !=
4554 NVME_CHANGED_NSID_SIZE
) {
4556 * If more than 1024 namespaces, the first entry in the log page should
4557 * be set to FFFFFFFFh and the others to 0 as spec.
4559 if (i
== ARRAY_SIZE(nslist
)) {
4560 memset(nslist
, 0x0, sizeof(nslist
));
4561 nslist
[0] = 0xffffffff;
4566 clear_bit(nsid
, n
->changed_nsids
);
4570 * Remove all the remaining list entries in case returns directly due to
4571 * more than 1024 namespaces.
4573 if (nslist
[0] == 0xffffffff) {
4574 bitmap_zero(n
->changed_nsids
, NVME_CHANGED_NSID_SIZE
);
4578 nvme_clear_events(n
, NVME_AER_TYPE_NOTICE
);
4581 return nvme_c2h(n
, ((uint8_t *)nslist
) + off
, trans_len
, req
);
4584 static uint16_t nvme_cmd_effects(NvmeCtrl
*n
, uint8_t csi
, uint32_t buf_len
,
4585 uint64_t off
, NvmeRequest
*req
)
4587 NvmeEffectsLog log
= {};
4588 const uint32_t *src_iocs
= NULL
;
4591 if (off
>= sizeof(log
)) {
4592 trace_pci_nvme_err_invalid_log_page_offset(off
, sizeof(log
));
4593 return NVME_INVALID_FIELD
| NVME_DNR
;
4596 switch (NVME_CC_CSS(ldl_le_p(&n
->bar
.cc
))) {
4597 case NVME_CC_CSS_NVM
:
4598 src_iocs
= nvme_cse_iocs_nvm
;
4600 case NVME_CC_CSS_ADMIN_ONLY
:
4602 case NVME_CC_CSS_CSI
:
4605 src_iocs
= nvme_cse_iocs_nvm
;
4607 case NVME_CSI_ZONED
:
4608 src_iocs
= nvme_cse_iocs_zoned
;
4613 memcpy(log
.acs
, nvme_cse_acs
, sizeof(nvme_cse_acs
));
4616 memcpy(log
.iocs
, src_iocs
, sizeof(log
.iocs
));
4619 trans_len
= MIN(sizeof(log
) - off
, buf_len
);
4621 return nvme_c2h(n
, ((uint8_t *)&log
) + off
, trans_len
, req
);
4624 static uint16_t nvme_get_log(NvmeCtrl
*n
, NvmeRequest
*req
)
4626 NvmeCmd
*cmd
= &req
->cmd
;
4628 uint32_t dw10
= le32_to_cpu(cmd
->cdw10
);
4629 uint32_t dw11
= le32_to_cpu(cmd
->cdw11
);
4630 uint32_t dw12
= le32_to_cpu(cmd
->cdw12
);
4631 uint32_t dw13
= le32_to_cpu(cmd
->cdw13
);
4632 uint8_t lid
= dw10
& 0xff;
4633 uint8_t lsp
= (dw10
>> 8) & 0xf;
4634 uint8_t rae
= (dw10
>> 15) & 0x1;
4635 uint8_t csi
= le32_to_cpu(cmd
->cdw14
) >> 24;
4636 uint32_t numdl
, numdu
;
4637 uint64_t off
, lpol
, lpou
;
4641 numdl
= (dw10
>> 16);
4642 numdu
= (dw11
& 0xffff);
4646 len
= (((numdu
<< 16) | numdl
) + 1) << 2;
4647 off
= (lpou
<< 32ULL) | lpol
;
4650 return NVME_INVALID_FIELD
| NVME_DNR
;
4653 trace_pci_nvme_get_log(nvme_cid(req
), lid
, lsp
, rae
, len
, off
);
4655 status
= nvme_check_mdts(n
, len
);
4661 case NVME_LOG_ERROR_INFO
:
4662 return nvme_error_info(n
, rae
, len
, off
, req
);
4663 case NVME_LOG_SMART_INFO
:
4664 return nvme_smart_info(n
, rae
, len
, off
, req
);
4665 case NVME_LOG_FW_SLOT_INFO
:
4666 return nvme_fw_log_info(n
, len
, off
, req
);
4667 case NVME_LOG_CHANGED_NSLIST
:
4668 return nvme_changed_nslist(n
, rae
, len
, off
, req
);
4669 case NVME_LOG_CMD_EFFECTS
:
4670 return nvme_cmd_effects(n
, csi
, len
, off
, req
);
4671 case NVME_LOG_ENDGRP
:
4672 return nvme_endgrp_info(n
, rae
, len
, off
, req
);
4674 trace_pci_nvme_err_invalid_log_page(nvme_cid(req
), lid
);
4675 return NVME_INVALID_FIELD
| NVME_DNR
;
4679 static void nvme_free_cq(NvmeCQueue
*cq
, NvmeCtrl
*n
)
4681 PCIDevice
*pci
= PCI_DEVICE(n
);
4682 uint16_t offset
= (cq
->cqid
<< 3) + (1 << 2);
4684 n
->cq
[cq
->cqid
] = NULL
;
4685 qemu_bh_delete(cq
->bh
);
4686 if (cq
->ioeventfd_enabled
) {
4687 memory_region_del_eventfd(&n
->iomem
,
4688 0x1000 + offset
, 4, false, 0, &cq
->notifier
);
4689 event_notifier_set_handler(&cq
->notifier
, NULL
);
4690 event_notifier_cleanup(&cq
->notifier
);
4692 if (msix_enabled(pci
)) {
4693 msix_vector_unuse(pci
, cq
->vector
);
4700 static uint16_t nvme_del_cq(NvmeCtrl
*n
, NvmeRequest
*req
)
4702 NvmeDeleteQ
*c
= (NvmeDeleteQ
*)&req
->cmd
;
4704 uint16_t qid
= le16_to_cpu(c
->qid
);
4706 if (unlikely(!qid
|| nvme_check_cqid(n
, qid
))) {
4707 trace_pci_nvme_err_invalid_del_cq_cqid(qid
);
4708 return NVME_INVALID_CQID
| NVME_DNR
;
4712 if (unlikely(!QTAILQ_EMPTY(&cq
->sq_list
))) {
4713 trace_pci_nvme_err_invalid_del_cq_notempty(qid
);
4714 return NVME_INVALID_QUEUE_DEL
;
4717 if (cq
->irq_enabled
&& cq
->tail
!= cq
->head
) {
4721 nvme_irq_deassert(n
, cq
);
4722 trace_pci_nvme_del_cq(qid
);
4723 nvme_free_cq(cq
, n
);
4724 return NVME_SUCCESS
;
4727 static void nvme_init_cq(NvmeCQueue
*cq
, NvmeCtrl
*n
, uint64_t dma_addr
,
4728 uint16_t cqid
, uint16_t vector
, uint16_t size
,
4729 uint16_t irq_enabled
)
4731 PCIDevice
*pci
= PCI_DEVICE(n
);
4733 if (msix_enabled(pci
)) {
4734 msix_vector_use(pci
, vector
);
4739 cq
->dma_addr
= dma_addr
;
4741 cq
->irq_enabled
= irq_enabled
;
4742 cq
->vector
= vector
;
4743 cq
->head
= cq
->tail
= 0;
4744 QTAILQ_INIT(&cq
->req_list
);
4745 QTAILQ_INIT(&cq
->sq_list
);
4746 if (n
->dbbuf_enabled
) {
4747 cq
->db_addr
= n
->dbbuf_dbs
+ (cqid
<< 3) + (1 << 2);
4748 cq
->ei_addr
= n
->dbbuf_eis
+ (cqid
<< 3) + (1 << 2);
4750 if (n
->params
.ioeventfd
&& cqid
!= 0) {
4751 if (!nvme_init_cq_ioeventfd(cq
)) {
4752 cq
->ioeventfd_enabled
= true;
4757 cq
->bh
= qemu_bh_new(nvme_post_cqes
, cq
);
4760 static uint16_t nvme_create_cq(NvmeCtrl
*n
, NvmeRequest
*req
)
4763 NvmeCreateCq
*c
= (NvmeCreateCq
*)&req
->cmd
;
4764 uint16_t cqid
= le16_to_cpu(c
->cqid
);
4765 uint16_t vector
= le16_to_cpu(c
->irq_vector
);
4766 uint16_t qsize
= le16_to_cpu(c
->qsize
);
4767 uint16_t qflags
= le16_to_cpu(c
->cq_flags
);
4768 uint64_t prp1
= le64_to_cpu(c
->prp1
);
4770 trace_pci_nvme_create_cq(prp1
, cqid
, vector
, qsize
, qflags
,
4771 NVME_CQ_FLAGS_IEN(qflags
) != 0);
4773 if (unlikely(!cqid
|| cqid
> n
->conf_ioqpairs
|| n
->cq
[cqid
] != NULL
)) {
4774 trace_pci_nvme_err_invalid_create_cq_cqid(cqid
);
4775 return NVME_INVALID_QID
| NVME_DNR
;
4777 if (unlikely(!qsize
|| qsize
> NVME_CAP_MQES(ldq_le_p(&n
->bar
.cap
)))) {
4778 trace_pci_nvme_err_invalid_create_cq_size(qsize
);
4779 return NVME_MAX_QSIZE_EXCEEDED
| NVME_DNR
;
4781 if (unlikely(prp1
& (n
->page_size
- 1))) {
4782 trace_pci_nvme_err_invalid_create_cq_addr(prp1
);
4783 return NVME_INVALID_PRP_OFFSET
| NVME_DNR
;
4785 if (unlikely(!msix_enabled(PCI_DEVICE(n
)) && vector
)) {
4786 trace_pci_nvme_err_invalid_create_cq_vector(vector
);
4787 return NVME_INVALID_IRQ_VECTOR
| NVME_DNR
;
4789 if (unlikely(vector
>= n
->conf_msix_qsize
)) {
4790 trace_pci_nvme_err_invalid_create_cq_vector(vector
);
4791 return NVME_INVALID_IRQ_VECTOR
| NVME_DNR
;
4793 if (unlikely(!(NVME_CQ_FLAGS_PC(qflags
)))) {
4794 trace_pci_nvme_err_invalid_create_cq_qflags(NVME_CQ_FLAGS_PC(qflags
));
4795 return NVME_INVALID_FIELD
| NVME_DNR
;
4798 cq
= g_malloc0(sizeof(*cq
));
4799 nvme_init_cq(cq
, n
, prp1
, cqid
, vector
, qsize
+ 1,
4800 NVME_CQ_FLAGS_IEN(qflags
));
4803 * It is only required to set qs_created when creating a completion queue;
4804 * creating a submission queue without a matching completion queue will
4807 n
->qs_created
= true;
4808 return NVME_SUCCESS
;
4811 static uint16_t nvme_rpt_empty_id_struct(NvmeCtrl
*n
, NvmeRequest
*req
)
4813 uint8_t id
[NVME_IDENTIFY_DATA_SIZE
] = {};
4815 return nvme_c2h(n
, id
, sizeof(id
), req
);
4818 static uint16_t nvme_identify_ctrl(NvmeCtrl
*n
, NvmeRequest
*req
)
4820 trace_pci_nvme_identify_ctrl();
4822 return nvme_c2h(n
, (uint8_t *)&n
->id_ctrl
, sizeof(n
->id_ctrl
), req
);
4825 static uint16_t nvme_identify_ctrl_csi(NvmeCtrl
*n
, NvmeRequest
*req
)
4827 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
4828 uint8_t id
[NVME_IDENTIFY_DATA_SIZE
] = {};
4829 NvmeIdCtrlNvm
*id_nvm
= (NvmeIdCtrlNvm
*)&id
;
4831 trace_pci_nvme_identify_ctrl_csi(c
->csi
);
4835 id_nvm
->vsl
= n
->params
.vsl
;
4836 id_nvm
->dmrsl
= cpu_to_le32(n
->dmrsl
);
4839 case NVME_CSI_ZONED
:
4840 ((NvmeIdCtrlZoned
*)&id
)->zasl
= n
->params
.zasl
;
4844 return NVME_INVALID_FIELD
| NVME_DNR
;
4847 return nvme_c2h(n
, id
, sizeof(id
), req
);
4850 static uint16_t nvme_identify_ns(NvmeCtrl
*n
, NvmeRequest
*req
, bool active
)
4853 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
4854 uint32_t nsid
= le32_to_cpu(c
->nsid
);
4856 trace_pci_nvme_identify_ns(nsid
);
4858 if (!nvme_nsid_valid(n
, nsid
) || nsid
== NVME_NSID_BROADCAST
) {
4859 return NVME_INVALID_NSID
| NVME_DNR
;
4862 ns
= nvme_ns(n
, nsid
);
4863 if (unlikely(!ns
)) {
4865 ns
= nvme_subsys_ns(n
->subsys
, nsid
);
4867 return nvme_rpt_empty_id_struct(n
, req
);
4870 return nvme_rpt_empty_id_struct(n
, req
);
4874 if (active
|| ns
->csi
== NVME_CSI_NVM
) {
4875 return nvme_c2h(n
, (uint8_t *)&ns
->id_ns
, sizeof(NvmeIdNs
), req
);
4878 return NVME_INVALID_CMD_SET
| NVME_DNR
;
4881 static uint16_t nvme_identify_ctrl_list(NvmeCtrl
*n
, NvmeRequest
*req
,
4884 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
4885 uint32_t nsid
= le32_to_cpu(c
->nsid
);
4886 uint16_t min_id
= le16_to_cpu(c
->ctrlid
);
4887 uint16_t list
[NVME_CONTROLLER_LIST_SIZE
] = {};
4888 uint16_t *ids
= &list
[1];
4891 int cntlid
, nr_ids
= 0;
4893 trace_pci_nvme_identify_ctrl_list(c
->cns
, min_id
);
4896 return NVME_INVALID_FIELD
| NVME_DNR
;
4900 if (nsid
== NVME_NSID_BROADCAST
) {
4901 return NVME_INVALID_FIELD
| NVME_DNR
;
4904 ns
= nvme_subsys_ns(n
->subsys
, nsid
);
4906 return NVME_INVALID_FIELD
| NVME_DNR
;
4910 for (cntlid
= min_id
; cntlid
< ARRAY_SIZE(n
->subsys
->ctrls
); cntlid
++) {
4911 ctrl
= nvme_subsys_ctrl(n
->subsys
, cntlid
);
4916 if (attached
&& !nvme_ns(ctrl
, nsid
)) {
4920 ids
[nr_ids
++] = cntlid
;
4925 return nvme_c2h(n
, (uint8_t *)list
, sizeof(list
), req
);
4928 static uint16_t nvme_identify_pri_ctrl_cap(NvmeCtrl
*n
, NvmeRequest
*req
)
4930 trace_pci_nvme_identify_pri_ctrl_cap(le16_to_cpu(n
->pri_ctrl_cap
.cntlid
));
4932 return nvme_c2h(n
, (uint8_t *)&n
->pri_ctrl_cap
,
4933 sizeof(NvmePriCtrlCap
), req
);
4936 static uint16_t nvme_identify_sec_ctrl_list(NvmeCtrl
*n
, NvmeRequest
*req
)
4938 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
4939 uint16_t pri_ctrl_id
= le16_to_cpu(n
->pri_ctrl_cap
.cntlid
);
4940 uint16_t min_id
= le16_to_cpu(c
->ctrlid
);
4941 uint8_t num_sec_ctrl
= n
->sec_ctrl_list
.numcntl
;
4942 NvmeSecCtrlList list
= {0};
4945 for (i
= 0; i
< num_sec_ctrl
; i
++) {
4946 if (n
->sec_ctrl_list
.sec
[i
].scid
>= min_id
) {
4947 list
.numcntl
= num_sec_ctrl
- i
;
4948 memcpy(&list
.sec
, n
->sec_ctrl_list
.sec
+ i
,
4949 list
.numcntl
* sizeof(NvmeSecCtrlEntry
));
4954 trace_pci_nvme_identify_sec_ctrl_list(pri_ctrl_id
, list
.numcntl
);
4956 return nvme_c2h(n
, (uint8_t *)&list
, sizeof(list
), req
);
4959 static uint16_t nvme_identify_ns_csi(NvmeCtrl
*n
, NvmeRequest
*req
,
4963 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
4964 uint32_t nsid
= le32_to_cpu(c
->nsid
);
4966 trace_pci_nvme_identify_ns_csi(nsid
, c
->csi
);
4968 if (!nvme_nsid_valid(n
, nsid
) || nsid
== NVME_NSID_BROADCAST
) {
4969 return NVME_INVALID_NSID
| NVME_DNR
;
4972 ns
= nvme_ns(n
, nsid
);
4973 if (unlikely(!ns
)) {
4975 ns
= nvme_subsys_ns(n
->subsys
, nsid
);
4977 return nvme_rpt_empty_id_struct(n
, req
);
4980 return nvme_rpt_empty_id_struct(n
, req
);
4984 if (c
->csi
== NVME_CSI_NVM
) {
4985 return nvme_c2h(n
, (uint8_t *)&ns
->id_ns_nvm
, sizeof(NvmeIdNsNvm
),
4987 } else if (c
->csi
== NVME_CSI_ZONED
&& ns
->csi
== NVME_CSI_ZONED
) {
4988 return nvme_c2h(n
, (uint8_t *)ns
->id_ns_zoned
, sizeof(NvmeIdNsZoned
),
4992 return NVME_INVALID_FIELD
| NVME_DNR
;
4995 static uint16_t nvme_identify_nslist(NvmeCtrl
*n
, NvmeRequest
*req
,
4999 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
5000 uint32_t min_nsid
= le32_to_cpu(c
->nsid
);
5001 uint8_t list
[NVME_IDENTIFY_DATA_SIZE
] = {};
5002 static const int data_len
= sizeof(list
);
5003 uint32_t *list_ptr
= (uint32_t *)list
;
5006 trace_pci_nvme_identify_nslist(min_nsid
);
5009 * Both FFFFFFFFh (NVME_NSID_BROADCAST) and FFFFFFFFEh are invalid values
5010 * since the Active Namespace ID List should return namespaces with ids
5011 * *higher* than the NSID specified in the command. This is also specified
5012 * in the spec (NVM Express v1.3d, Section 5.15.4).
5014 if (min_nsid
>= NVME_NSID_BROADCAST
- 1) {
5015 return NVME_INVALID_NSID
| NVME_DNR
;
5018 for (i
= 1; i
<= NVME_MAX_NAMESPACES
; i
++) {
5022 ns
= nvme_subsys_ns(n
->subsys
, i
);
5030 if (ns
->params
.nsid
<= min_nsid
) {
5033 list_ptr
[j
++] = cpu_to_le32(ns
->params
.nsid
);
5034 if (j
== data_len
/ sizeof(uint32_t)) {
5039 return nvme_c2h(n
, list
, data_len
, req
);
5042 static uint16_t nvme_identify_nslist_csi(NvmeCtrl
*n
, NvmeRequest
*req
,
5046 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
5047 uint32_t min_nsid
= le32_to_cpu(c
->nsid
);
5048 uint8_t list
[NVME_IDENTIFY_DATA_SIZE
] = {};
5049 static const int data_len
= sizeof(list
);
5050 uint32_t *list_ptr
= (uint32_t *)list
;
5053 trace_pci_nvme_identify_nslist_csi(min_nsid
, c
->csi
);
5056 * Same as in nvme_identify_nslist(), FFFFFFFFh/FFFFFFFFEh are invalid.
5058 if (min_nsid
>= NVME_NSID_BROADCAST
- 1) {
5059 return NVME_INVALID_NSID
| NVME_DNR
;
5062 if (c
->csi
!= NVME_CSI_NVM
&& c
->csi
!= NVME_CSI_ZONED
) {
5063 return NVME_INVALID_FIELD
| NVME_DNR
;
5066 for (i
= 1; i
<= NVME_MAX_NAMESPACES
; i
++) {
5070 ns
= nvme_subsys_ns(n
->subsys
, i
);
5078 if (ns
->params
.nsid
<= min_nsid
|| c
->csi
!= ns
->csi
) {
5081 list_ptr
[j
++] = cpu_to_le32(ns
->params
.nsid
);
5082 if (j
== data_len
/ sizeof(uint32_t)) {
5087 return nvme_c2h(n
, list
, data_len
, req
);
5090 static uint16_t nvme_identify_ns_descr_list(NvmeCtrl
*n
, NvmeRequest
*req
)
5093 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
5094 uint32_t nsid
= le32_to_cpu(c
->nsid
);
5095 uint8_t list
[NVME_IDENTIFY_DATA_SIZE
] = {};
5096 uint8_t *pos
= list
;
5099 uint8_t v
[NVME_NIDL_UUID
];
5100 } QEMU_PACKED uuid
= {};
5104 } QEMU_PACKED eui64
= {};
5108 } QEMU_PACKED csi
= {};
5110 trace_pci_nvme_identify_ns_descr_list(nsid
);
5112 if (!nvme_nsid_valid(n
, nsid
) || nsid
== NVME_NSID_BROADCAST
) {
5113 return NVME_INVALID_NSID
| NVME_DNR
;
5116 ns
= nvme_ns(n
, nsid
);
5117 if (unlikely(!ns
)) {
5118 return NVME_INVALID_FIELD
| NVME_DNR
;
5121 if (!qemu_uuid_is_null(&ns
->params
.uuid
)) {
5122 uuid
.hdr
.nidt
= NVME_NIDT_UUID
;
5123 uuid
.hdr
.nidl
= NVME_NIDL_UUID
;
5124 memcpy(uuid
.v
, ns
->params
.uuid
.data
, NVME_NIDL_UUID
);
5125 memcpy(pos
, &uuid
, sizeof(uuid
));
5126 pos
+= sizeof(uuid
);
5129 if (ns
->params
.eui64
) {
5130 eui64
.hdr
.nidt
= NVME_NIDT_EUI64
;
5131 eui64
.hdr
.nidl
= NVME_NIDL_EUI64
;
5132 eui64
.v
= cpu_to_be64(ns
->params
.eui64
);
5133 memcpy(pos
, &eui64
, sizeof(eui64
));
5134 pos
+= sizeof(eui64
);
5137 csi
.hdr
.nidt
= NVME_NIDT_CSI
;
5138 csi
.hdr
.nidl
= NVME_NIDL_CSI
;
5140 memcpy(pos
, &csi
, sizeof(csi
));
5143 return nvme_c2h(n
, list
, sizeof(list
), req
);
5146 static uint16_t nvme_identify_cmd_set(NvmeCtrl
*n
, NvmeRequest
*req
)
5148 uint8_t list
[NVME_IDENTIFY_DATA_SIZE
] = {};
5149 static const int data_len
= sizeof(list
);
5151 trace_pci_nvme_identify_cmd_set();
5153 NVME_SET_CSI(*list
, NVME_CSI_NVM
);
5154 NVME_SET_CSI(*list
, NVME_CSI_ZONED
);
5156 return nvme_c2h(n
, list
, data_len
, req
);
5159 static uint16_t nvme_identify(NvmeCtrl
*n
, NvmeRequest
*req
)
5161 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
5163 trace_pci_nvme_identify(nvme_cid(req
), c
->cns
, le16_to_cpu(c
->ctrlid
),
5167 case NVME_ID_CNS_NS
:
5168 return nvme_identify_ns(n
, req
, true);
5169 case NVME_ID_CNS_NS_PRESENT
:
5170 return nvme_identify_ns(n
, req
, false);
5171 case NVME_ID_CNS_NS_ATTACHED_CTRL_LIST
:
5172 return nvme_identify_ctrl_list(n
, req
, true);
5173 case NVME_ID_CNS_CTRL_LIST
:
5174 return nvme_identify_ctrl_list(n
, req
, false);
5175 case NVME_ID_CNS_PRIMARY_CTRL_CAP
:
5176 return nvme_identify_pri_ctrl_cap(n
, req
);
5177 case NVME_ID_CNS_SECONDARY_CTRL_LIST
:
5178 return nvme_identify_sec_ctrl_list(n
, req
);
5179 case NVME_ID_CNS_CS_NS
:
5180 return nvme_identify_ns_csi(n
, req
, true);
5181 case NVME_ID_CNS_CS_NS_PRESENT
:
5182 return nvme_identify_ns_csi(n
, req
, false);
5183 case NVME_ID_CNS_CTRL
:
5184 return nvme_identify_ctrl(n
, req
);
5185 case NVME_ID_CNS_CS_CTRL
:
5186 return nvme_identify_ctrl_csi(n
, req
);
5187 case NVME_ID_CNS_NS_ACTIVE_LIST
:
5188 return nvme_identify_nslist(n
, req
, true);
5189 case NVME_ID_CNS_NS_PRESENT_LIST
:
5190 return nvme_identify_nslist(n
, req
, false);
5191 case NVME_ID_CNS_CS_NS_ACTIVE_LIST
:
5192 return nvme_identify_nslist_csi(n
, req
, true);
5193 case NVME_ID_CNS_CS_NS_PRESENT_LIST
:
5194 return nvme_identify_nslist_csi(n
, req
, false);
5195 case NVME_ID_CNS_NS_DESCR_LIST
:
5196 return nvme_identify_ns_descr_list(n
, req
);
5197 case NVME_ID_CNS_IO_COMMAND_SET
:
5198 return nvme_identify_cmd_set(n
, req
);
5200 trace_pci_nvme_err_invalid_identify_cns(le32_to_cpu(c
->cns
));
5201 return NVME_INVALID_FIELD
| NVME_DNR
;
5205 static uint16_t nvme_abort(NvmeCtrl
*n
, NvmeRequest
*req
)
5207 uint16_t sqid
= le32_to_cpu(req
->cmd
.cdw10
) & 0xffff;
5209 req
->cqe
.result
= 1;
5210 if (nvme_check_sqid(n
, sqid
)) {
5211 return NVME_INVALID_FIELD
| NVME_DNR
;
5214 return NVME_SUCCESS
;
5217 static inline void nvme_set_timestamp(NvmeCtrl
*n
, uint64_t ts
)
5219 trace_pci_nvme_setfeat_timestamp(ts
);
5221 n
->host_timestamp
= le64_to_cpu(ts
);
5222 n
->timestamp_set_qemu_clock_ms
= qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
);
5225 static inline uint64_t nvme_get_timestamp(const NvmeCtrl
*n
)
5227 uint64_t current_time
= qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
);
5228 uint64_t elapsed_time
= current_time
- n
->timestamp_set_qemu_clock_ms
;
5230 union nvme_timestamp
{
5232 uint64_t timestamp
:48;
5240 union nvme_timestamp ts
;
5242 ts
.timestamp
= n
->host_timestamp
+ elapsed_time
;
5244 /* If the host timestamp is non-zero, set the timestamp origin */
5245 ts
.origin
= n
->host_timestamp
? 0x01 : 0x00;
5247 trace_pci_nvme_getfeat_timestamp(ts
.all
);
5249 return cpu_to_le64(ts
.all
);
5252 static uint16_t nvme_get_feature_timestamp(NvmeCtrl
*n
, NvmeRequest
*req
)
5254 uint64_t timestamp
= nvme_get_timestamp(n
);
5256 return nvme_c2h(n
, (uint8_t *)×tamp
, sizeof(timestamp
), req
);
5259 static uint16_t nvme_get_feature(NvmeCtrl
*n
, NvmeRequest
*req
)
5261 NvmeCmd
*cmd
= &req
->cmd
;
5262 uint32_t dw10
= le32_to_cpu(cmd
->cdw10
);
5263 uint32_t dw11
= le32_to_cpu(cmd
->cdw11
);
5264 uint32_t nsid
= le32_to_cpu(cmd
->nsid
);
5266 uint8_t fid
= NVME_GETSETFEAT_FID(dw10
);
5267 NvmeGetFeatureSelect sel
= NVME_GETFEAT_SELECT(dw10
);
5272 static const uint32_t nvme_feature_default
[NVME_FID_MAX
] = {
5273 [NVME_ARBITRATION
] = NVME_ARB_AB_NOLIMIT
,
5276 trace_pci_nvme_getfeat(nvme_cid(req
), nsid
, fid
, sel
, dw11
);
5278 if (!nvme_feature_support
[fid
]) {
5279 return NVME_INVALID_FIELD
| NVME_DNR
;
5282 if (nvme_feature_cap
[fid
] & NVME_FEAT_CAP_NS
) {
5283 if (!nvme_nsid_valid(n
, nsid
) || nsid
== NVME_NSID_BROADCAST
) {
5285 * The Reservation Notification Mask and Reservation Persistence
5286 * features require a status code of Invalid Field in Command when
5287 * NSID is FFFFFFFFh. Since the device does not support those
5288 * features we can always return Invalid Namespace or Format as we
5289 * should do for all other features.
5291 return NVME_INVALID_NSID
| NVME_DNR
;
5294 if (!nvme_ns(n
, nsid
)) {
5295 return NVME_INVALID_FIELD
| NVME_DNR
;
5300 case NVME_GETFEAT_SELECT_CURRENT
:
5302 case NVME_GETFEAT_SELECT_SAVED
:
5303 /* no features are saveable by the controller; fallthrough */
5304 case NVME_GETFEAT_SELECT_DEFAULT
:
5306 case NVME_GETFEAT_SELECT_CAP
:
5307 result
= nvme_feature_cap
[fid
];
5312 case NVME_TEMPERATURE_THRESHOLD
:
5316 * The controller only implements the Composite Temperature sensor, so
5317 * return 0 for all other sensors.
5319 if (NVME_TEMP_TMPSEL(dw11
) != NVME_TEMP_TMPSEL_COMPOSITE
) {
5323 switch (NVME_TEMP_THSEL(dw11
)) {
5324 case NVME_TEMP_THSEL_OVER
:
5325 result
= n
->features
.temp_thresh_hi
;
5327 case NVME_TEMP_THSEL_UNDER
:
5328 result
= n
->features
.temp_thresh_low
;
5332 return NVME_INVALID_FIELD
| NVME_DNR
;
5333 case NVME_ERROR_RECOVERY
:
5334 if (!nvme_nsid_valid(n
, nsid
)) {
5335 return NVME_INVALID_NSID
| NVME_DNR
;
5338 ns
= nvme_ns(n
, nsid
);
5339 if (unlikely(!ns
)) {
5340 return NVME_INVALID_FIELD
| NVME_DNR
;
5343 result
= ns
->features
.err_rec
;
5345 case NVME_VOLATILE_WRITE_CACHE
:
5347 for (i
= 1; i
<= NVME_MAX_NAMESPACES
; i
++) {
5353 result
= blk_enable_write_cache(ns
->blkconf
.blk
);
5358 trace_pci_nvme_getfeat_vwcache(result
? "enabled" : "disabled");
5360 case NVME_ASYNCHRONOUS_EVENT_CONF
:
5361 result
= n
->features
.async_config
;
5363 case NVME_TIMESTAMP
:
5364 return nvme_get_feature_timestamp(n
, req
);
5365 case NVME_HOST_BEHAVIOR_SUPPORT
:
5366 return nvme_c2h(n
, (uint8_t *)&n
->features
.hbs
,
5367 sizeof(n
->features
.hbs
), req
);
5374 case NVME_TEMPERATURE_THRESHOLD
:
5377 if (NVME_TEMP_TMPSEL(dw11
) != NVME_TEMP_TMPSEL_COMPOSITE
) {
5381 if (NVME_TEMP_THSEL(dw11
) == NVME_TEMP_THSEL_OVER
) {
5382 result
= NVME_TEMPERATURE_WARNING
;
5386 case NVME_NUMBER_OF_QUEUES
:
5387 result
= (n
->conf_ioqpairs
- 1) | ((n
->conf_ioqpairs
- 1) << 16);
5388 trace_pci_nvme_getfeat_numq(result
);
5390 case NVME_INTERRUPT_VECTOR_CONF
:
5392 if (iv
>= n
->conf_ioqpairs
+ 1) {
5393 return NVME_INVALID_FIELD
| NVME_DNR
;
5397 if (iv
== n
->admin_cq
.vector
) {
5398 result
|= NVME_INTVC_NOCOALESCING
;
5402 result
= nvme_feature_default
[fid
];
5407 req
->cqe
.result
= cpu_to_le32(result
);
5408 return NVME_SUCCESS
;
5411 static uint16_t nvme_set_feature_timestamp(NvmeCtrl
*n
, NvmeRequest
*req
)
5416 ret
= nvme_h2c(n
, (uint8_t *)×tamp
, sizeof(timestamp
), req
);
5421 nvme_set_timestamp(n
, timestamp
);
5423 return NVME_SUCCESS
;
5426 static uint16_t nvme_set_feature(NvmeCtrl
*n
, NvmeRequest
*req
)
5428 NvmeNamespace
*ns
= NULL
;
5430 NvmeCmd
*cmd
= &req
->cmd
;
5431 uint32_t dw10
= le32_to_cpu(cmd
->cdw10
);
5432 uint32_t dw11
= le32_to_cpu(cmd
->cdw11
);
5433 uint32_t nsid
= le32_to_cpu(cmd
->nsid
);
5434 uint8_t fid
= NVME_GETSETFEAT_FID(dw10
);
5435 uint8_t save
= NVME_SETFEAT_SAVE(dw10
);
5439 trace_pci_nvme_setfeat(nvme_cid(req
), nsid
, fid
, save
, dw11
);
5441 if (save
&& !(nvme_feature_cap
[fid
] & NVME_FEAT_CAP_SAVE
)) {
5442 return NVME_FID_NOT_SAVEABLE
| NVME_DNR
;
5445 if (!nvme_feature_support
[fid
]) {
5446 return NVME_INVALID_FIELD
| NVME_DNR
;
5449 if (nvme_feature_cap
[fid
] & NVME_FEAT_CAP_NS
) {
5450 if (nsid
!= NVME_NSID_BROADCAST
) {
5451 if (!nvme_nsid_valid(n
, nsid
)) {
5452 return NVME_INVALID_NSID
| NVME_DNR
;
5455 ns
= nvme_ns(n
, nsid
);
5456 if (unlikely(!ns
)) {
5457 return NVME_INVALID_FIELD
| NVME_DNR
;
5460 } else if (nsid
&& nsid
!= NVME_NSID_BROADCAST
) {
5461 if (!nvme_nsid_valid(n
, nsid
)) {
5462 return NVME_INVALID_NSID
| NVME_DNR
;
5465 return NVME_FEAT_NOT_NS_SPEC
| NVME_DNR
;
5468 if (!(nvme_feature_cap
[fid
] & NVME_FEAT_CAP_CHANGE
)) {
5469 return NVME_FEAT_NOT_CHANGEABLE
| NVME_DNR
;
5473 case NVME_TEMPERATURE_THRESHOLD
:
5474 if (NVME_TEMP_TMPSEL(dw11
) != NVME_TEMP_TMPSEL_COMPOSITE
) {
5478 switch (NVME_TEMP_THSEL(dw11
)) {
5479 case NVME_TEMP_THSEL_OVER
:
5480 n
->features
.temp_thresh_hi
= NVME_TEMP_TMPTH(dw11
);
5482 case NVME_TEMP_THSEL_UNDER
:
5483 n
->features
.temp_thresh_low
= NVME_TEMP_TMPTH(dw11
);
5486 return NVME_INVALID_FIELD
| NVME_DNR
;
5489 if ((n
->temperature
>= n
->features
.temp_thresh_hi
) ||
5490 (n
->temperature
<= n
->features
.temp_thresh_low
)) {
5491 nvme_smart_event(n
, NVME_SMART_TEMPERATURE
);
5495 case NVME_ERROR_RECOVERY
:
5496 if (nsid
== NVME_NSID_BROADCAST
) {
5497 for (i
= 1; i
<= NVME_MAX_NAMESPACES
; i
++) {
5504 if (NVME_ID_NS_NSFEAT_DULBE(ns
->id_ns
.nsfeat
)) {
5505 ns
->features
.err_rec
= dw11
;
5513 if (NVME_ID_NS_NSFEAT_DULBE(ns
->id_ns
.nsfeat
)) {
5514 ns
->features
.err_rec
= dw11
;
5517 case NVME_VOLATILE_WRITE_CACHE
:
5518 for (i
= 1; i
<= NVME_MAX_NAMESPACES
; i
++) {
5524 if (!(dw11
& 0x1) && blk_enable_write_cache(ns
->blkconf
.blk
)) {
5525 blk_flush(ns
->blkconf
.blk
);
5528 blk_set_enable_write_cache(ns
->blkconf
.blk
, dw11
& 1);
5533 case NVME_NUMBER_OF_QUEUES
:
5534 if (n
->qs_created
) {
5535 return NVME_CMD_SEQ_ERROR
| NVME_DNR
;
5539 * NVMe v1.3, Section 5.21.1.7: FFFFh is not an allowed value for NCQR
5542 if ((dw11
& 0xffff) == 0xffff || ((dw11
>> 16) & 0xffff) == 0xffff) {
5543 return NVME_INVALID_FIELD
| NVME_DNR
;
5546 trace_pci_nvme_setfeat_numq((dw11
& 0xffff) + 1,
5547 ((dw11
>> 16) & 0xffff) + 1,
5550 req
->cqe
.result
= cpu_to_le32((n
->conf_ioqpairs
- 1) |
5551 ((n
->conf_ioqpairs
- 1) << 16));
5553 case NVME_ASYNCHRONOUS_EVENT_CONF
:
5554 n
->features
.async_config
= dw11
;
5556 case NVME_TIMESTAMP
:
5557 return nvme_set_feature_timestamp(n
, req
);
5558 case NVME_HOST_BEHAVIOR_SUPPORT
:
5559 status
= nvme_h2c(n
, (uint8_t *)&n
->features
.hbs
,
5560 sizeof(n
->features
.hbs
), req
);
5565 for (i
= 1; i
<= NVME_MAX_NAMESPACES
; i
++) {
5572 ns
->id_ns
.nlbaf
= ns
->nlbaf
- 1;
5573 if (!n
->features
.hbs
.lbafee
) {
5574 ns
->id_ns
.nlbaf
= MIN(ns
->id_ns
.nlbaf
, 15);
5579 case NVME_COMMAND_SET_PROFILE
:
5581 trace_pci_nvme_err_invalid_iocsci(dw11
& 0x1ff);
5582 return NVME_CMD_SET_CMB_REJECTED
| NVME_DNR
;
5586 return NVME_FEAT_NOT_CHANGEABLE
| NVME_DNR
;
5588 return NVME_SUCCESS
;
5591 static uint16_t nvme_aer(NvmeCtrl
*n
, NvmeRequest
*req
)
5593 trace_pci_nvme_aer(nvme_cid(req
));
5595 if (n
->outstanding_aers
> n
->params
.aerl
) {
5596 trace_pci_nvme_aer_aerl_exceeded();
5597 return NVME_AER_LIMIT_EXCEEDED
;
5600 n
->aer_reqs
[n
->outstanding_aers
] = req
;
5601 n
->outstanding_aers
++;
5603 if (!QTAILQ_EMPTY(&n
->aer_queue
)) {
5604 nvme_process_aers(n
);
5607 return NVME_NO_COMPLETE
;
5610 static void nvme_update_dmrsl(NvmeCtrl
*n
)
5614 for (nsid
= 1; nsid
<= NVME_MAX_NAMESPACES
; nsid
++) {
5615 NvmeNamespace
*ns
= nvme_ns(n
, nsid
);
5620 n
->dmrsl
= MIN_NON_ZERO(n
->dmrsl
,
5621 BDRV_REQUEST_MAX_BYTES
/ nvme_l2b(ns
, 1));
5625 static void nvme_select_iocs_ns(NvmeCtrl
*n
, NvmeNamespace
*ns
)
5627 uint32_t cc
= ldl_le_p(&n
->bar
.cc
);
5629 ns
->iocs
= nvme_cse_iocs_none
;
5632 if (NVME_CC_CSS(cc
) != NVME_CC_CSS_ADMIN_ONLY
) {
5633 ns
->iocs
= nvme_cse_iocs_nvm
;
5636 case NVME_CSI_ZONED
:
5637 if (NVME_CC_CSS(cc
) == NVME_CC_CSS_CSI
) {
5638 ns
->iocs
= nvme_cse_iocs_zoned
;
5639 } else if (NVME_CC_CSS(cc
) == NVME_CC_CSS_NVM
) {
5640 ns
->iocs
= nvme_cse_iocs_nvm
;
5646 static uint16_t nvme_ns_attachment(NvmeCtrl
*n
, NvmeRequest
*req
)
5650 uint16_t list
[NVME_CONTROLLER_LIST_SIZE
] = {};
5651 uint32_t nsid
= le32_to_cpu(req
->cmd
.nsid
);
5652 uint32_t dw10
= le32_to_cpu(req
->cmd
.cdw10
);
5653 uint8_t sel
= dw10
& 0xf;
5654 uint16_t *nr_ids
= &list
[0];
5655 uint16_t *ids
= &list
[1];
5659 trace_pci_nvme_ns_attachment(nvme_cid(req
), dw10
& 0xf);
5661 if (!nvme_nsid_valid(n
, nsid
)) {
5662 return NVME_INVALID_NSID
| NVME_DNR
;
5665 ns
= nvme_subsys_ns(n
->subsys
, nsid
);
5667 return NVME_INVALID_FIELD
| NVME_DNR
;
5670 ret
= nvme_h2c(n
, (uint8_t *)list
, 4096, req
);
5676 return NVME_NS_CTRL_LIST_INVALID
| NVME_DNR
;
5679 *nr_ids
= MIN(*nr_ids
, NVME_CONTROLLER_LIST_SIZE
- 1);
5680 for (i
= 0; i
< *nr_ids
; i
++) {
5681 ctrl
= nvme_subsys_ctrl(n
->subsys
, ids
[i
]);
5683 return NVME_NS_CTRL_LIST_INVALID
| NVME_DNR
;
5687 case NVME_NS_ATTACHMENT_ATTACH
:
5688 if (nvme_ns(ctrl
, nsid
)) {
5689 return NVME_NS_ALREADY_ATTACHED
| NVME_DNR
;
5692 if (ns
->attached
&& !ns
->params
.shared
) {
5693 return NVME_NS_PRIVATE
| NVME_DNR
;
5696 nvme_attach_ns(ctrl
, ns
);
5697 nvme_select_iocs_ns(ctrl
, ns
);
5701 case NVME_NS_ATTACHMENT_DETACH
:
5702 if (!nvme_ns(ctrl
, nsid
)) {
5703 return NVME_NS_NOT_ATTACHED
| NVME_DNR
;
5706 ctrl
->namespaces
[nsid
] = NULL
;
5709 nvme_update_dmrsl(ctrl
);
5714 return NVME_INVALID_FIELD
| NVME_DNR
;
5718 * Add namespace id to the changed namespace id list for event clearing
5719 * via Get Log Page command.
5721 if (!test_and_set_bit(nsid
, ctrl
->changed_nsids
)) {
5722 nvme_enqueue_event(ctrl
, NVME_AER_TYPE_NOTICE
,
5723 NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED
,
5724 NVME_LOG_CHANGED_NSLIST
);
5728 return NVME_SUCCESS
;
5731 typedef struct NvmeFormatAIOCB
{
5748 static void nvme_format_cancel(BlockAIOCB
*aiocb
)
5750 NvmeFormatAIOCB
*iocb
= container_of(aiocb
, NvmeFormatAIOCB
, common
);
5752 iocb
->ret
= -ECANCELED
;
5755 blk_aio_cancel_async(iocb
->aiocb
);
5760 static const AIOCBInfo nvme_format_aiocb_info
= {
5761 .aiocb_size
= sizeof(NvmeFormatAIOCB
),
5762 .cancel_async
= nvme_format_cancel
,
5763 .get_aio_context
= nvme_get_aio_context
,
5766 static void nvme_format_set(NvmeNamespace
*ns
, uint8_t lbaf
, uint8_t mset
,
5767 uint8_t pi
, uint8_t pil
)
5769 uint8_t lbafl
= lbaf
& 0xf;
5770 uint8_t lbafu
= lbaf
>> 4;
5772 trace_pci_nvme_format_set(ns
->params
.nsid
, lbaf
, mset
, pi
, pil
);
5774 ns
->id_ns
.dps
= (pil
<< 3) | pi
;
5775 ns
->id_ns
.flbas
= (lbafu
<< 5) | (mset
<< 4) | lbafl
;
5777 nvme_ns_init_format(ns
);
5780 static void nvme_do_format(NvmeFormatAIOCB
*iocb
);
5782 static void nvme_format_ns_cb(void *opaque
, int ret
)
5784 NvmeFormatAIOCB
*iocb
= opaque
;
5785 NvmeNamespace
*ns
= iocb
->ns
;
5788 if (iocb
->ret
< 0) {
5790 } else if (ret
< 0) {
5797 if (iocb
->offset
< ns
->size
) {
5798 bytes
= MIN(BDRV_REQUEST_MAX_BYTES
, ns
->size
- iocb
->offset
);
5800 iocb
->aiocb
= blk_aio_pwrite_zeroes(ns
->blkconf
.blk
, iocb
->offset
,
5801 bytes
, BDRV_REQ_MAY_UNMAP
,
5802 nvme_format_ns_cb
, iocb
);
5804 iocb
->offset
+= bytes
;
5808 nvme_format_set(ns
, iocb
->lbaf
, iocb
->mset
, iocb
->pi
, iocb
->pil
);
5814 nvme_do_format(iocb
);
5817 static uint16_t nvme_format_check(NvmeNamespace
*ns
, uint8_t lbaf
, uint8_t pi
)
5819 if (ns
->params
.zoned
) {
5820 return NVME_INVALID_FORMAT
| NVME_DNR
;
5823 if (lbaf
> ns
->id_ns
.nlbaf
) {
5824 return NVME_INVALID_FORMAT
| NVME_DNR
;
5827 if (pi
&& (ns
->id_ns
.lbaf
[lbaf
].ms
< nvme_pi_tuple_size(ns
))) {
5828 return NVME_INVALID_FORMAT
| NVME_DNR
;
5831 if (pi
&& pi
> NVME_ID_NS_DPS_TYPE_3
) {
5832 return NVME_INVALID_FIELD
| NVME_DNR
;
5835 return NVME_SUCCESS
;
5838 static void nvme_do_format(NvmeFormatAIOCB
*iocb
)
5840 NvmeRequest
*req
= iocb
->req
;
5841 NvmeCtrl
*n
= nvme_ctrl(req
);
5842 uint32_t dw10
= le32_to_cpu(req
->cmd
.cdw10
);
5843 uint8_t lbaf
= dw10
& 0xf;
5844 uint8_t pi
= (dw10
>> 5) & 0x7;
5848 if (iocb
->ret
< 0) {
5852 if (iocb
->broadcast
) {
5853 for (i
= iocb
->nsid
+ 1; i
<= NVME_MAX_NAMESPACES
; i
++) {
5854 iocb
->ns
= nvme_ns(n
, i
);
5866 status
= nvme_format_check(iocb
->ns
, lbaf
, pi
);
5868 req
->status
= status
;
5872 iocb
->ns
->status
= NVME_FORMAT_IN_PROGRESS
;
5873 nvme_format_ns_cb(iocb
, 0);
5877 iocb
->common
.cb(iocb
->common
.opaque
, iocb
->ret
);
5878 qemu_aio_unref(iocb
);
5881 static uint16_t nvme_format(NvmeCtrl
*n
, NvmeRequest
*req
)
5883 NvmeFormatAIOCB
*iocb
;
5884 uint32_t nsid
= le32_to_cpu(req
->cmd
.nsid
);
5885 uint32_t dw10
= le32_to_cpu(req
->cmd
.cdw10
);
5886 uint8_t lbaf
= dw10
& 0xf;
5887 uint8_t mset
= (dw10
>> 4) & 0x1;
5888 uint8_t pi
= (dw10
>> 5) & 0x7;
5889 uint8_t pil
= (dw10
>> 8) & 0x1;
5890 uint8_t lbafu
= (dw10
>> 12) & 0x3;
5893 iocb
= qemu_aio_get(&nvme_format_aiocb_info
, NULL
, nvme_misc_cb
, req
);
5903 iocb
->broadcast
= (nsid
== NVME_NSID_BROADCAST
);
5906 if (n
->features
.hbs
.lbafee
) {
5907 iocb
->lbaf
|= lbafu
<< 4;
5910 if (!iocb
->broadcast
) {
5911 if (!nvme_nsid_valid(n
, nsid
)) {
5912 status
= NVME_INVALID_NSID
| NVME_DNR
;
5916 iocb
->ns
= nvme_ns(n
, nsid
);
5918 status
= NVME_INVALID_FIELD
| NVME_DNR
;
5923 req
->aiocb
= &iocb
->common
;
5924 nvme_do_format(iocb
);
5926 return NVME_NO_COMPLETE
;
5929 qemu_aio_unref(iocb
);
5934 static void nvme_get_virt_res_num(NvmeCtrl
*n
, uint8_t rt
, int *num_total
,
5935 int *num_prim
, int *num_sec
)
5937 *num_total
= le32_to_cpu(rt
?
5938 n
->pri_ctrl_cap
.vifrt
: n
->pri_ctrl_cap
.vqfrt
);
5939 *num_prim
= le16_to_cpu(rt
?
5940 n
->pri_ctrl_cap
.virfap
: n
->pri_ctrl_cap
.vqrfap
);
5941 *num_sec
= le16_to_cpu(rt
? n
->pri_ctrl_cap
.virfa
: n
->pri_ctrl_cap
.vqrfa
);
5944 static uint16_t nvme_assign_virt_res_to_prim(NvmeCtrl
*n
, NvmeRequest
*req
,
5945 uint16_t cntlid
, uint8_t rt
,
5948 int num_total
, num_prim
, num_sec
;
5950 if (cntlid
!= n
->cntlid
) {
5951 return NVME_INVALID_CTRL_ID
| NVME_DNR
;
5954 nvme_get_virt_res_num(n
, rt
, &num_total
, &num_prim
, &num_sec
);
5956 if (nr
> num_total
) {
5957 return NVME_INVALID_NUM_RESOURCES
| NVME_DNR
;
5960 if (nr
> num_total
- num_sec
) {
5961 return NVME_INVALID_RESOURCE_ID
| NVME_DNR
;
5965 n
->next_pri_ctrl_cap
.virfap
= cpu_to_le16(nr
);
5967 n
->next_pri_ctrl_cap
.vqrfap
= cpu_to_le16(nr
);
5970 req
->cqe
.result
= cpu_to_le32(nr
);
5974 static void nvme_update_virt_res(NvmeCtrl
*n
, NvmeSecCtrlEntry
*sctrl
,
5977 int prev_nr
, prev_total
;
5980 prev_nr
= le16_to_cpu(sctrl
->nvi
);
5981 prev_total
= le32_to_cpu(n
->pri_ctrl_cap
.virfa
);
5982 sctrl
->nvi
= cpu_to_le16(nr
);
5983 n
->pri_ctrl_cap
.virfa
= cpu_to_le32(prev_total
+ nr
- prev_nr
);
5985 prev_nr
= le16_to_cpu(sctrl
->nvq
);
5986 prev_total
= le32_to_cpu(n
->pri_ctrl_cap
.vqrfa
);
5987 sctrl
->nvq
= cpu_to_le16(nr
);
5988 n
->pri_ctrl_cap
.vqrfa
= cpu_to_le32(prev_total
+ nr
- prev_nr
);
5992 static uint16_t nvme_assign_virt_res_to_sec(NvmeCtrl
*n
, NvmeRequest
*req
,
5993 uint16_t cntlid
, uint8_t rt
, int nr
)
5995 int num_total
, num_prim
, num_sec
, num_free
, diff
, limit
;
5996 NvmeSecCtrlEntry
*sctrl
;
5998 sctrl
= nvme_sctrl_for_cntlid(n
, cntlid
);
6000 return NVME_INVALID_CTRL_ID
| NVME_DNR
;
6004 return NVME_INVALID_SEC_CTRL_STATE
| NVME_DNR
;
6007 limit
= le16_to_cpu(rt
? n
->pri_ctrl_cap
.vifrsm
: n
->pri_ctrl_cap
.vqfrsm
);
6009 return NVME_INVALID_NUM_RESOURCES
| NVME_DNR
;
6012 nvme_get_virt_res_num(n
, rt
, &num_total
, &num_prim
, &num_sec
);
6013 num_free
= num_total
- num_prim
- num_sec
;
6014 diff
= nr
- le16_to_cpu(rt
? sctrl
->nvi
: sctrl
->nvq
);
6016 if (diff
> num_free
) {
6017 return NVME_INVALID_RESOURCE_ID
| NVME_DNR
;
6020 nvme_update_virt_res(n
, sctrl
, rt
, nr
);
6021 req
->cqe
.result
= cpu_to_le32(nr
);
6026 static uint16_t nvme_virt_set_state(NvmeCtrl
*n
, uint16_t cntlid
, bool online
)
6028 PCIDevice
*pci
= PCI_DEVICE(n
);
6029 NvmeCtrl
*sn
= NULL
;
6030 NvmeSecCtrlEntry
*sctrl
;
6033 sctrl
= nvme_sctrl_for_cntlid(n
, cntlid
);
6035 return NVME_INVALID_CTRL_ID
| NVME_DNR
;
6038 if (!pci_is_vf(pci
)) {
6039 vf_index
= le16_to_cpu(sctrl
->vfn
) - 1;
6040 sn
= NVME(pcie_sriov_get_vf_at_index(pci
, vf_index
));
6044 if (!sctrl
->nvi
|| (le16_to_cpu(sctrl
->nvq
) < 2) || !sn
) {
6045 return NVME_INVALID_SEC_CTRL_STATE
| NVME_DNR
;
6050 nvme_ctrl_reset(sn
, NVME_RESET_FUNCTION
);
6053 nvme_update_virt_res(n
, sctrl
, NVME_VIRT_RES_INTERRUPT
, 0);
6054 nvme_update_virt_res(n
, sctrl
, NVME_VIRT_RES_QUEUE
, 0);
6059 nvme_ctrl_reset(sn
, NVME_RESET_FUNCTION
);
6064 return NVME_SUCCESS
;
6067 static uint16_t nvme_virt_mngmt(NvmeCtrl
*n
, NvmeRequest
*req
)
6069 uint32_t dw10
= le32_to_cpu(req
->cmd
.cdw10
);
6070 uint32_t dw11
= le32_to_cpu(req
->cmd
.cdw11
);
6071 uint8_t act
= dw10
& 0xf;
6072 uint8_t rt
= (dw10
>> 8) & 0x7;
6073 uint16_t cntlid
= (dw10
>> 16) & 0xffff;
6074 int nr
= dw11
& 0xffff;
6076 trace_pci_nvme_virt_mngmt(nvme_cid(req
), act
, cntlid
, rt
? "VI" : "VQ", nr
);
6078 if (rt
!= NVME_VIRT_RES_QUEUE
&& rt
!= NVME_VIRT_RES_INTERRUPT
) {
6079 return NVME_INVALID_RESOURCE_ID
| NVME_DNR
;
6083 case NVME_VIRT_MNGMT_ACTION_SEC_ASSIGN
:
6084 return nvme_assign_virt_res_to_sec(n
, req
, cntlid
, rt
, nr
);
6085 case NVME_VIRT_MNGMT_ACTION_PRM_ALLOC
:
6086 return nvme_assign_virt_res_to_prim(n
, req
, cntlid
, rt
, nr
);
6087 case NVME_VIRT_MNGMT_ACTION_SEC_ONLINE
:
6088 return nvme_virt_set_state(n
, cntlid
, true);
6089 case NVME_VIRT_MNGMT_ACTION_SEC_OFFLINE
:
6090 return nvme_virt_set_state(n
, cntlid
, false);
6092 return NVME_INVALID_FIELD
| NVME_DNR
;
6096 static uint16_t nvme_dbbuf_config(NvmeCtrl
*n
, const NvmeRequest
*req
)
6098 PCIDevice
*pci
= PCI_DEVICE(n
);
6099 uint64_t dbs_addr
= le64_to_cpu(req
->cmd
.dptr
.prp1
);
6100 uint64_t eis_addr
= le64_to_cpu(req
->cmd
.dptr
.prp2
);
6103 /* Address should be page aligned */
6104 if (dbs_addr
& (n
->page_size
- 1) || eis_addr
& (n
->page_size
- 1)) {
6105 return NVME_INVALID_FIELD
| NVME_DNR
;
6108 /* Save shadow buffer base addr for use during queue creation */
6109 n
->dbbuf_dbs
= dbs_addr
;
6110 n
->dbbuf_eis
= eis_addr
;
6111 n
->dbbuf_enabled
= true;
6113 for (i
= 0; i
< n
->params
.max_ioqpairs
+ 1; i
++) {
6114 NvmeSQueue
*sq
= n
->sq
[i
];
6115 NvmeCQueue
*cq
= n
->cq
[i
];
6119 * CAP.DSTRD is 0, so offset of ith sq db_addr is (i<<3)
6120 * nvme_process_db() uses this hard-coded way to calculate
6121 * doorbell offsets. Be consistent with that here.
6123 sq
->db_addr
= dbs_addr
+ (i
<< 3);
6124 sq
->ei_addr
= eis_addr
+ (i
<< 3);
6125 pci_dma_write(pci
, sq
->db_addr
, &sq
->tail
, sizeof(sq
->tail
));
6127 if (n
->params
.ioeventfd
&& sq
->sqid
!= 0) {
6128 if (!nvme_init_sq_ioeventfd(sq
)) {
6129 sq
->ioeventfd_enabled
= true;
6135 /* CAP.DSTRD is 0, so offset of ith cq db_addr is (i<<3)+(1<<2) */
6136 cq
->db_addr
= dbs_addr
+ (i
<< 3) + (1 << 2);
6137 cq
->ei_addr
= eis_addr
+ (i
<< 3) + (1 << 2);
6138 pci_dma_write(pci
, cq
->db_addr
, &cq
->head
, sizeof(cq
->head
));
6140 if (n
->params
.ioeventfd
&& cq
->cqid
!= 0) {
6141 if (!nvme_init_cq_ioeventfd(cq
)) {
6142 cq
->ioeventfd_enabled
= true;
6148 trace_pci_nvme_dbbuf_config(dbs_addr
, eis_addr
);
6150 return NVME_SUCCESS
;
6153 static uint16_t nvme_admin_cmd(NvmeCtrl
*n
, NvmeRequest
*req
)
6155 trace_pci_nvme_admin_cmd(nvme_cid(req
), nvme_sqid(req
), req
->cmd
.opcode
,
6156 nvme_adm_opc_str(req
->cmd
.opcode
));
6158 if (!(nvme_cse_acs
[req
->cmd
.opcode
] & NVME_CMD_EFF_CSUPP
)) {
6159 trace_pci_nvme_err_invalid_admin_opc(req
->cmd
.opcode
);
6160 return NVME_INVALID_OPCODE
| NVME_DNR
;
6163 /* SGLs shall not be used for Admin commands in NVMe over PCIe */
6164 if (NVME_CMD_FLAGS_PSDT(req
->cmd
.flags
) != NVME_PSDT_PRP
) {
6165 return NVME_INVALID_FIELD
| NVME_DNR
;
6168 if (NVME_CMD_FLAGS_FUSE(req
->cmd
.flags
)) {
6169 return NVME_INVALID_FIELD
;
6172 switch (req
->cmd
.opcode
) {
6173 case NVME_ADM_CMD_DELETE_SQ
:
6174 return nvme_del_sq(n
, req
);
6175 case NVME_ADM_CMD_CREATE_SQ
:
6176 return nvme_create_sq(n
, req
);
6177 case NVME_ADM_CMD_GET_LOG_PAGE
:
6178 return nvme_get_log(n
, req
);
6179 case NVME_ADM_CMD_DELETE_CQ
:
6180 return nvme_del_cq(n
, req
);
6181 case NVME_ADM_CMD_CREATE_CQ
:
6182 return nvme_create_cq(n
, req
);
6183 case NVME_ADM_CMD_IDENTIFY
:
6184 return nvme_identify(n
, req
);
6185 case NVME_ADM_CMD_ABORT
:
6186 return nvme_abort(n
, req
);
6187 case NVME_ADM_CMD_SET_FEATURES
:
6188 return nvme_set_feature(n
, req
);
6189 case NVME_ADM_CMD_GET_FEATURES
:
6190 return nvme_get_feature(n
, req
);
6191 case NVME_ADM_CMD_ASYNC_EV_REQ
:
6192 return nvme_aer(n
, req
);
6193 case NVME_ADM_CMD_NS_ATTACHMENT
:
6194 return nvme_ns_attachment(n
, req
);
6195 case NVME_ADM_CMD_VIRT_MNGMT
:
6196 return nvme_virt_mngmt(n
, req
);
6197 case NVME_ADM_CMD_DBBUF_CONFIG
:
6198 return nvme_dbbuf_config(n
, req
);
6199 case NVME_ADM_CMD_FORMAT_NVM
:
6200 return nvme_format(n
, req
);
6205 return NVME_INVALID_OPCODE
| NVME_DNR
;
6208 static void nvme_update_sq_eventidx(const NvmeSQueue
*sq
)
6210 uint32_t v
= cpu_to_le32(sq
->tail
);
6212 trace_pci_nvme_update_sq_eventidx(sq
->sqid
, sq
->tail
);
6214 pci_dma_write(PCI_DEVICE(sq
->ctrl
), sq
->ei_addr
, &v
, sizeof(v
));
6217 static void nvme_update_sq_tail(NvmeSQueue
*sq
)
6221 pci_dma_read(PCI_DEVICE(sq
->ctrl
), sq
->db_addr
, &v
, sizeof(v
));
6223 sq
->tail
= le32_to_cpu(v
);
6225 trace_pci_nvme_update_sq_tail(sq
->sqid
, sq
->tail
);
6228 static void nvme_process_sq(void *opaque
)
6230 NvmeSQueue
*sq
= opaque
;
6231 NvmeCtrl
*n
= sq
->ctrl
;
6232 NvmeCQueue
*cq
= n
->cq
[sq
->cqid
];
6239 if (n
->dbbuf_enabled
) {
6240 nvme_update_sq_tail(sq
);
6243 while (!(nvme_sq_empty(sq
) || QTAILQ_EMPTY(&sq
->req_list
))) {
6244 addr
= sq
->dma_addr
+ sq
->head
* n
->sqe_size
;
6245 if (nvme_addr_read(n
, addr
, (void *)&cmd
, sizeof(cmd
))) {
6246 trace_pci_nvme_err_addr_read(addr
);
6247 trace_pci_nvme_err_cfs();
6248 stl_le_p(&n
->bar
.csts
, NVME_CSTS_FAILED
);
6251 nvme_inc_sq_head(sq
);
6253 req
= QTAILQ_FIRST(&sq
->req_list
);
6254 QTAILQ_REMOVE(&sq
->req_list
, req
, entry
);
6255 QTAILQ_INSERT_TAIL(&sq
->out_req_list
, req
, entry
);
6256 nvme_req_clear(req
);
6257 req
->cqe
.cid
= cmd
.cid
;
6258 memcpy(&req
->cmd
, &cmd
, sizeof(NvmeCmd
));
6260 status
= sq
->sqid
? nvme_io_cmd(n
, req
) :
6261 nvme_admin_cmd(n
, req
);
6262 if (status
!= NVME_NO_COMPLETE
) {
6263 req
->status
= status
;
6264 nvme_enqueue_req_completion(cq
, req
);
6267 if (n
->dbbuf_enabled
) {
6268 nvme_update_sq_eventidx(sq
);
6269 nvme_update_sq_tail(sq
);
6274 static void nvme_update_msixcap_ts(PCIDevice
*pci_dev
, uint32_t table_size
)
6278 if (!msix_present(pci_dev
)) {
6282 assert(table_size
> 0 && table_size
<= pci_dev
->msix_entries_nr
);
6284 config
= pci_dev
->config
+ pci_dev
->msix_cap
;
6285 pci_set_word_by_mask(config
+ PCI_MSIX_FLAGS
, PCI_MSIX_FLAGS_QSIZE
,
6289 static void nvme_activate_virt_res(NvmeCtrl
*n
)
6291 PCIDevice
*pci_dev
= PCI_DEVICE(n
);
6292 NvmePriCtrlCap
*cap
= &n
->pri_ctrl_cap
;
6293 NvmeSecCtrlEntry
*sctrl
;
6295 /* -1 to account for the admin queue */
6296 if (pci_is_vf(pci_dev
)) {
6297 sctrl
= nvme_sctrl(n
);
6298 cap
->vqprt
= sctrl
->nvq
;
6299 cap
->viprt
= sctrl
->nvi
;
6300 n
->conf_ioqpairs
= sctrl
->nvq
? le16_to_cpu(sctrl
->nvq
) - 1 : 0;
6301 n
->conf_msix_qsize
= sctrl
->nvi
? le16_to_cpu(sctrl
->nvi
) : 1;
6303 cap
->vqrfap
= n
->next_pri_ctrl_cap
.vqrfap
;
6304 cap
->virfap
= n
->next_pri_ctrl_cap
.virfap
;
6305 n
->conf_ioqpairs
= le16_to_cpu(cap
->vqprt
) +
6306 le16_to_cpu(cap
->vqrfap
) - 1;
6307 n
->conf_msix_qsize
= le16_to_cpu(cap
->viprt
) +
6308 le16_to_cpu(cap
->virfap
);
6312 static void nvme_ctrl_reset(NvmeCtrl
*n
, NvmeResetType rst
)
6314 PCIDevice
*pci_dev
= PCI_DEVICE(n
);
6315 NvmeSecCtrlEntry
*sctrl
;
6319 for (i
= 1; i
<= NVME_MAX_NAMESPACES
; i
++) {
6328 for (i
= 0; i
< n
->params
.max_ioqpairs
+ 1; i
++) {
6329 if (n
->sq
[i
] != NULL
) {
6330 nvme_free_sq(n
->sq
[i
], n
);
6333 for (i
= 0; i
< n
->params
.max_ioqpairs
+ 1; i
++) {
6334 if (n
->cq
[i
] != NULL
) {
6335 nvme_free_cq(n
->cq
[i
], n
);
6339 while (!QTAILQ_EMPTY(&n
->aer_queue
)) {
6340 NvmeAsyncEvent
*event
= QTAILQ_FIRST(&n
->aer_queue
);
6341 QTAILQ_REMOVE(&n
->aer_queue
, event
, entry
);
6345 if (n
->params
.sriov_max_vfs
) {
6346 if (!pci_is_vf(pci_dev
)) {
6347 for (i
= 0; i
< n
->sec_ctrl_list
.numcntl
; i
++) {
6348 sctrl
= &n
->sec_ctrl_list
.sec
[i
];
6349 nvme_virt_set_state(n
, le16_to_cpu(sctrl
->scid
), false);
6352 if (rst
!= NVME_RESET_CONTROLLER
) {
6353 pcie_sriov_pf_disable_vfs(pci_dev
);
6357 if (rst
!= NVME_RESET_CONTROLLER
) {
6358 nvme_activate_virt_res(n
);
6364 n
->outstanding_aers
= 0;
6365 n
->qs_created
= false;
6367 nvme_update_msixcap_ts(pci_dev
, n
->conf_msix_qsize
);
6369 if (pci_is_vf(pci_dev
)) {
6370 sctrl
= nvme_sctrl(n
);
6372 stl_le_p(&n
->bar
.csts
, sctrl
->scs
? 0 : NVME_CSTS_FAILED
);
6374 stl_le_p(&n
->bar
.csts
, 0);
6377 stl_le_p(&n
->bar
.intms
, 0);
6378 stl_le_p(&n
->bar
.intmc
, 0);
6379 stl_le_p(&n
->bar
.cc
, 0);
6383 n
->dbbuf_enabled
= false;
6386 static void nvme_ctrl_shutdown(NvmeCtrl
*n
)
6392 memory_region_msync(&n
->pmr
.dev
->mr
, 0, n
->pmr
.dev
->size
);
6395 for (i
= 1; i
<= NVME_MAX_NAMESPACES
; i
++) {
6401 nvme_ns_shutdown(ns
);
6405 static void nvme_select_iocs(NvmeCtrl
*n
)
6410 for (i
= 1; i
<= NVME_MAX_NAMESPACES
; i
++) {
6416 nvme_select_iocs_ns(n
, ns
);
6420 static int nvme_start_ctrl(NvmeCtrl
*n
)
6422 uint64_t cap
= ldq_le_p(&n
->bar
.cap
);
6423 uint32_t cc
= ldl_le_p(&n
->bar
.cc
);
6424 uint32_t aqa
= ldl_le_p(&n
->bar
.aqa
);
6425 uint64_t asq
= ldq_le_p(&n
->bar
.asq
);
6426 uint64_t acq
= ldq_le_p(&n
->bar
.acq
);
6427 uint32_t page_bits
= NVME_CC_MPS(cc
) + 12;
6428 uint32_t page_size
= 1 << page_bits
;
6429 NvmeSecCtrlEntry
*sctrl
= nvme_sctrl(n
);
6431 if (pci_is_vf(PCI_DEVICE(n
)) && !sctrl
->scs
) {
6432 trace_pci_nvme_err_startfail_virt_state(le16_to_cpu(sctrl
->nvi
),
6433 le16_to_cpu(sctrl
->nvq
),
6434 sctrl
->scs
? "ONLINE" :
6438 if (unlikely(n
->cq
[0])) {
6439 trace_pci_nvme_err_startfail_cq();
6442 if (unlikely(n
->sq
[0])) {
6443 trace_pci_nvme_err_startfail_sq();
6446 if (unlikely(asq
& (page_size
- 1))) {
6447 trace_pci_nvme_err_startfail_asq_misaligned(asq
);
6450 if (unlikely(acq
& (page_size
- 1))) {
6451 trace_pci_nvme_err_startfail_acq_misaligned(acq
);
6454 if (unlikely(!(NVME_CAP_CSS(cap
) & (1 << NVME_CC_CSS(cc
))))) {
6455 trace_pci_nvme_err_startfail_css(NVME_CC_CSS(cc
));
6458 if (unlikely(NVME_CC_MPS(cc
) < NVME_CAP_MPSMIN(cap
))) {
6459 trace_pci_nvme_err_startfail_page_too_small(
6461 NVME_CAP_MPSMIN(cap
));
6464 if (unlikely(NVME_CC_MPS(cc
) >
6465 NVME_CAP_MPSMAX(cap
))) {
6466 trace_pci_nvme_err_startfail_page_too_large(
6468 NVME_CAP_MPSMAX(cap
));
6471 if (unlikely(NVME_CC_IOCQES(cc
) <
6472 NVME_CTRL_CQES_MIN(n
->id_ctrl
.cqes
))) {
6473 trace_pci_nvme_err_startfail_cqent_too_small(
6475 NVME_CTRL_CQES_MIN(cap
));
6478 if (unlikely(NVME_CC_IOCQES(cc
) >
6479 NVME_CTRL_CQES_MAX(n
->id_ctrl
.cqes
))) {
6480 trace_pci_nvme_err_startfail_cqent_too_large(
6482 NVME_CTRL_CQES_MAX(cap
));
6485 if (unlikely(NVME_CC_IOSQES(cc
) <
6486 NVME_CTRL_SQES_MIN(n
->id_ctrl
.sqes
))) {
6487 trace_pci_nvme_err_startfail_sqent_too_small(
6489 NVME_CTRL_SQES_MIN(cap
));
6492 if (unlikely(NVME_CC_IOSQES(cc
) >
6493 NVME_CTRL_SQES_MAX(n
->id_ctrl
.sqes
))) {
6494 trace_pci_nvme_err_startfail_sqent_too_large(
6496 NVME_CTRL_SQES_MAX(cap
));
6499 if (unlikely(!NVME_AQA_ASQS(aqa
))) {
6500 trace_pci_nvme_err_startfail_asqent_sz_zero();
6503 if (unlikely(!NVME_AQA_ACQS(aqa
))) {
6504 trace_pci_nvme_err_startfail_acqent_sz_zero();
6508 n
->page_bits
= page_bits
;
6509 n
->page_size
= page_size
;
6510 n
->max_prp_ents
= n
->page_size
/ sizeof(uint64_t);
6511 n
->cqe_size
= 1 << NVME_CC_IOCQES(cc
);
6512 n
->sqe_size
= 1 << NVME_CC_IOSQES(cc
);
6513 nvme_init_cq(&n
->admin_cq
, n
, acq
, 0, 0, NVME_AQA_ACQS(aqa
) + 1, 1);
6514 nvme_init_sq(&n
->admin_sq
, n
, asq
, 0, 0, NVME_AQA_ASQS(aqa
) + 1);
6516 nvme_set_timestamp(n
, 0ULL);
6518 nvme_select_iocs(n
);
6523 static void nvme_cmb_enable_regs(NvmeCtrl
*n
)
6525 uint32_t cmbloc
= ldl_le_p(&n
->bar
.cmbloc
);
6526 uint32_t cmbsz
= ldl_le_p(&n
->bar
.cmbsz
);
6528 NVME_CMBLOC_SET_CDPCILS(cmbloc
, 1);
6529 NVME_CMBLOC_SET_CDPMLS(cmbloc
, 1);
6530 NVME_CMBLOC_SET_BIR(cmbloc
, NVME_CMB_BIR
);
6531 stl_le_p(&n
->bar
.cmbloc
, cmbloc
);
6533 NVME_CMBSZ_SET_SQS(cmbsz
, 1);
6534 NVME_CMBSZ_SET_CQS(cmbsz
, 0);
6535 NVME_CMBSZ_SET_LISTS(cmbsz
, 1);
6536 NVME_CMBSZ_SET_RDS(cmbsz
, 1);
6537 NVME_CMBSZ_SET_WDS(cmbsz
, 1);
6538 NVME_CMBSZ_SET_SZU(cmbsz
, 2); /* MBs */
6539 NVME_CMBSZ_SET_SZ(cmbsz
, n
->params
.cmb_size_mb
);
6540 stl_le_p(&n
->bar
.cmbsz
, cmbsz
);
6543 static void nvme_write_bar(NvmeCtrl
*n
, hwaddr offset
, uint64_t data
,
6546 PCIDevice
*pci
= PCI_DEVICE(n
);
6547 uint64_t cap
= ldq_le_p(&n
->bar
.cap
);
6548 uint32_t cc
= ldl_le_p(&n
->bar
.cc
);
6549 uint32_t intms
= ldl_le_p(&n
->bar
.intms
);
6550 uint32_t csts
= ldl_le_p(&n
->bar
.csts
);
6551 uint32_t pmrsts
= ldl_le_p(&n
->bar
.pmrsts
);
6553 if (unlikely(offset
& (sizeof(uint32_t) - 1))) {
6554 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_misaligned32
,
6555 "MMIO write not 32-bit aligned,"
6556 " offset=0x%"PRIx64
"", offset
);
6557 /* should be ignored, fall through for now */
6560 if (unlikely(size
< sizeof(uint32_t))) {
6561 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_toosmall
,
6562 "MMIO write smaller than 32-bits,"
6563 " offset=0x%"PRIx64
", size=%u",
6565 /* should be ignored, fall through for now */
6569 case NVME_REG_INTMS
:
6570 if (unlikely(msix_enabled(pci
))) {
6571 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix
,
6572 "undefined access to interrupt mask set"
6573 " when MSI-X is enabled");
6574 /* should be ignored, fall through for now */
6577 stl_le_p(&n
->bar
.intms
, intms
);
6578 n
->bar
.intmc
= n
->bar
.intms
;
6579 trace_pci_nvme_mmio_intm_set(data
& 0xffffffff, intms
);
6582 case NVME_REG_INTMC
:
6583 if (unlikely(msix_enabled(pci
))) {
6584 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix
,
6585 "undefined access to interrupt mask clr"
6586 " when MSI-X is enabled");
6587 /* should be ignored, fall through for now */
6590 stl_le_p(&n
->bar
.intms
, intms
);
6591 n
->bar
.intmc
= n
->bar
.intms
;
6592 trace_pci_nvme_mmio_intm_clr(data
& 0xffffffff, intms
);
6596 stl_le_p(&n
->bar
.cc
, data
);
6598 trace_pci_nvme_mmio_cfg(data
& 0xffffffff);
6600 if (NVME_CC_SHN(data
) && !(NVME_CC_SHN(cc
))) {
6601 trace_pci_nvme_mmio_shutdown_set();
6602 nvme_ctrl_shutdown(n
);
6603 csts
&= ~(CSTS_SHST_MASK
<< CSTS_SHST_SHIFT
);
6604 csts
|= NVME_CSTS_SHST_COMPLETE
;
6605 } else if (!NVME_CC_SHN(data
) && NVME_CC_SHN(cc
)) {
6606 trace_pci_nvme_mmio_shutdown_cleared();
6607 csts
&= ~(CSTS_SHST_MASK
<< CSTS_SHST_SHIFT
);
6610 if (NVME_CC_EN(data
) && !NVME_CC_EN(cc
)) {
6611 if (unlikely(nvme_start_ctrl(n
))) {
6612 trace_pci_nvme_err_startfail();
6613 csts
= NVME_CSTS_FAILED
;
6615 trace_pci_nvme_mmio_start_success();
6616 csts
= NVME_CSTS_READY
;
6618 } else if (!NVME_CC_EN(data
) && NVME_CC_EN(cc
)) {
6619 trace_pci_nvme_mmio_stopped();
6620 nvme_ctrl_reset(n
, NVME_RESET_CONTROLLER
);
6625 stl_le_p(&n
->bar
.csts
, csts
);
6629 if (data
& (1 << 4)) {
6630 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ssreset_w1c_unsupported
,
6631 "attempted to W1C CSTS.NSSRO"
6632 " but CAP.NSSRS is zero (not supported)");
6633 } else if (data
!= 0) {
6634 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ro_csts
,
6635 "attempted to set a read only bit"
6636 " of controller status");
6640 if (data
== 0x4e564d65) {
6641 trace_pci_nvme_ub_mmiowr_ssreset_unsupported();
6643 /* The spec says that writes of other values have no effect */
6648 stl_le_p(&n
->bar
.aqa
, data
);
6649 trace_pci_nvme_mmio_aqattr(data
& 0xffffffff);
6652 stn_le_p(&n
->bar
.asq
, size
, data
);
6653 trace_pci_nvme_mmio_asqaddr(data
);
6655 case NVME_REG_ASQ
+ 4:
6656 stl_le_p((uint8_t *)&n
->bar
.asq
+ 4, data
);
6657 trace_pci_nvme_mmio_asqaddr_hi(data
, ldq_le_p(&n
->bar
.asq
));
6660 trace_pci_nvme_mmio_acqaddr(data
);
6661 stn_le_p(&n
->bar
.acq
, size
, data
);
6663 case NVME_REG_ACQ
+ 4:
6664 stl_le_p((uint8_t *)&n
->bar
.acq
+ 4, data
);
6665 trace_pci_nvme_mmio_acqaddr_hi(data
, ldq_le_p(&n
->bar
.acq
));
6667 case NVME_REG_CMBLOC
:
6668 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbloc_reserved
,
6669 "invalid write to reserved CMBLOC"
6670 " when CMBSZ is zero, ignored");
6672 case NVME_REG_CMBSZ
:
6673 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbsz_readonly
,
6674 "invalid write to read only CMBSZ, ignored");
6676 case NVME_REG_CMBMSC
:
6677 if (!NVME_CAP_CMBS(cap
)) {
6681 stn_le_p(&n
->bar
.cmbmsc
, size
, data
);
6682 n
->cmb
.cmse
= false;
6684 if (NVME_CMBMSC_CRE(data
)) {
6685 nvme_cmb_enable_regs(n
);
6687 if (NVME_CMBMSC_CMSE(data
)) {
6688 uint64_t cmbmsc
= ldq_le_p(&n
->bar
.cmbmsc
);
6689 hwaddr cba
= NVME_CMBMSC_CBA(cmbmsc
) << CMBMSC_CBA_SHIFT
;
6690 if (cba
+ int128_get64(n
->cmb
.mem
.size
) < cba
) {
6691 uint32_t cmbsts
= ldl_le_p(&n
->bar
.cmbsts
);
6692 NVME_CMBSTS_SET_CBAI(cmbsts
, 1);
6693 stl_le_p(&n
->bar
.cmbsts
, cmbsts
);
6706 case NVME_REG_CMBMSC
+ 4:
6707 stl_le_p((uint8_t *)&n
->bar
.cmbmsc
+ 4, data
);
6710 case NVME_REG_PMRCAP
:
6711 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrcap_readonly
,
6712 "invalid write to PMRCAP register, ignored");
6714 case NVME_REG_PMRCTL
:
6715 if (!NVME_CAP_PMRS(cap
)) {
6719 stl_le_p(&n
->bar
.pmrctl
, data
);
6720 if (NVME_PMRCTL_EN(data
)) {
6721 memory_region_set_enabled(&n
->pmr
.dev
->mr
, true);
6724 memory_region_set_enabled(&n
->pmr
.dev
->mr
, false);
6725 NVME_PMRSTS_SET_NRDY(pmrsts
, 1);
6726 n
->pmr
.cmse
= false;
6728 stl_le_p(&n
->bar
.pmrsts
, pmrsts
);
6730 case NVME_REG_PMRSTS
:
6731 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrsts_readonly
,
6732 "invalid write to PMRSTS register, ignored");
6734 case NVME_REG_PMREBS
:
6735 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrebs_readonly
,
6736 "invalid write to PMREBS register, ignored");
6738 case NVME_REG_PMRSWTP
:
6739 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrswtp_readonly
,
6740 "invalid write to PMRSWTP register, ignored");
6742 case NVME_REG_PMRMSCL
:
6743 if (!NVME_CAP_PMRS(cap
)) {
6747 stl_le_p(&n
->bar
.pmrmscl
, data
);
6748 n
->pmr
.cmse
= false;
6750 if (NVME_PMRMSCL_CMSE(data
)) {
6751 uint64_t pmrmscu
= ldl_le_p(&n
->bar
.pmrmscu
);
6752 hwaddr cba
= pmrmscu
<< 32 |
6753 (NVME_PMRMSCL_CBA(data
) << PMRMSCL_CBA_SHIFT
);
6754 if (cba
+ int128_get64(n
->pmr
.dev
->mr
.size
) < cba
) {
6755 NVME_PMRSTS_SET_CBAI(pmrsts
, 1);
6756 stl_le_p(&n
->bar
.pmrsts
, pmrsts
);
6765 case NVME_REG_PMRMSCU
:
6766 if (!NVME_CAP_PMRS(cap
)) {
6770 stl_le_p(&n
->bar
.pmrmscu
, data
);
6773 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_invalid
,
6774 "invalid MMIO write,"
6775 " offset=0x%"PRIx64
", data=%"PRIx64
"",
6781 static uint64_t nvme_mmio_read(void *opaque
, hwaddr addr
, unsigned size
)
6783 NvmeCtrl
*n
= (NvmeCtrl
*)opaque
;
6784 uint8_t *ptr
= (uint8_t *)&n
->bar
;
6786 trace_pci_nvme_mmio_read(addr
, size
);
6788 if (unlikely(addr
& (sizeof(uint32_t) - 1))) {
6789 NVME_GUEST_ERR(pci_nvme_ub_mmiord_misaligned32
,
6790 "MMIO read not 32-bit aligned,"
6791 " offset=0x%"PRIx64
"", addr
);
6792 /* should RAZ, fall through for now */
6793 } else if (unlikely(size
< sizeof(uint32_t))) {
6794 NVME_GUEST_ERR(pci_nvme_ub_mmiord_toosmall
,
6795 "MMIO read smaller than 32-bits,"
6796 " offset=0x%"PRIx64
"", addr
);
6797 /* should RAZ, fall through for now */
6800 if (addr
> sizeof(n
->bar
) - size
) {
6801 NVME_GUEST_ERR(pci_nvme_ub_mmiord_invalid_ofs
,
6802 "MMIO read beyond last register,"
6803 " offset=0x%"PRIx64
", returning 0", addr
);
6808 if (pci_is_vf(PCI_DEVICE(n
)) && !nvme_sctrl(n
)->scs
&&
6809 addr
!= NVME_REG_CSTS
) {
6810 trace_pci_nvme_err_ignored_mmio_vf_offline(addr
, size
);
6815 * When PMRWBM bit 1 is set then read from
6816 * from PMRSTS should ensure prior writes
6817 * made it to persistent media
6819 if (addr
== NVME_REG_PMRSTS
&&
6820 (NVME_PMRCAP_PMRWBM(ldl_le_p(&n
->bar
.pmrcap
)) & 0x02)) {
6821 memory_region_msync(&n
->pmr
.dev
->mr
, 0, n
->pmr
.dev
->size
);
6824 return ldn_le_p(ptr
+ addr
, size
);
6827 static void nvme_process_db(NvmeCtrl
*n
, hwaddr addr
, int val
)
6829 PCIDevice
*pci
= PCI_DEVICE(n
);
6832 if (unlikely(addr
& ((1 << 2) - 1))) {
6833 NVME_GUEST_ERR(pci_nvme_ub_db_wr_misaligned
,
6834 "doorbell write not 32-bit aligned,"
6835 " offset=0x%"PRIx64
", ignoring", addr
);
6839 if (((addr
- 0x1000) >> 2) & 1) {
6840 /* Completion queue doorbell write */
6842 uint16_t new_head
= val
& 0xffff;
6846 qid
= (addr
- (0x1000 + (1 << 2))) >> 3;
6847 if (unlikely(nvme_check_cqid(n
, qid
))) {
6848 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cq
,
6849 "completion queue doorbell write"
6850 " for nonexistent queue,"
6851 " sqid=%"PRIu32
", ignoring", qid
);
6854 * NVM Express v1.3d, Section 4.1 state: "If host software writes
6855 * an invalid value to the Submission Queue Tail Doorbell or
6856 * Completion Queue Head Doorbell regiter and an Asynchronous Event
6857 * Request command is outstanding, then an asynchronous event is
6858 * posted to the Admin Completion Queue with a status code of
6859 * Invalid Doorbell Write Value."
6861 * Also note that the spec includes the "Invalid Doorbell Register"
6862 * status code, but nowhere does it specify when to use it.
6863 * However, it seems reasonable to use it here in a similar
6866 if (n
->outstanding_aers
) {
6867 nvme_enqueue_event(n
, NVME_AER_TYPE_ERROR
,
6868 NVME_AER_INFO_ERR_INVALID_DB_REGISTER
,
6869 NVME_LOG_ERROR_INFO
);
6876 if (unlikely(new_head
>= cq
->size
)) {
6877 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cqhead
,
6878 "completion queue doorbell write value"
6879 " beyond queue size, sqid=%"PRIu32
","
6880 " new_head=%"PRIu16
", ignoring",
6883 if (n
->outstanding_aers
) {
6884 nvme_enqueue_event(n
, NVME_AER_TYPE_ERROR
,
6885 NVME_AER_INFO_ERR_INVALID_DB_VALUE
,
6886 NVME_LOG_ERROR_INFO
);
6892 trace_pci_nvme_mmio_doorbell_cq(cq
->cqid
, new_head
);
6894 start_sqs
= nvme_cq_full(cq
) ? 1 : 0;
6895 cq
->head
= new_head
;
6896 if (!qid
&& n
->dbbuf_enabled
) {
6897 pci_dma_write(pci
, cq
->db_addr
, &cq
->head
, sizeof(cq
->head
));
6901 QTAILQ_FOREACH(sq
, &cq
->sq_list
, entry
) {
6902 qemu_bh_schedule(sq
->bh
);
6904 qemu_bh_schedule(cq
->bh
);
6907 if (cq
->tail
== cq
->head
) {
6908 if (cq
->irq_enabled
) {
6912 nvme_irq_deassert(n
, cq
);
6915 /* Submission queue doorbell write */
6917 uint16_t new_tail
= val
& 0xffff;
6920 qid
= (addr
- 0x1000) >> 3;
6921 if (unlikely(nvme_check_sqid(n
, qid
))) {
6922 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sq
,
6923 "submission queue doorbell write"
6924 " for nonexistent queue,"
6925 " sqid=%"PRIu32
", ignoring", qid
);
6927 if (n
->outstanding_aers
) {
6928 nvme_enqueue_event(n
, NVME_AER_TYPE_ERROR
,
6929 NVME_AER_INFO_ERR_INVALID_DB_REGISTER
,
6930 NVME_LOG_ERROR_INFO
);
6937 if (unlikely(new_tail
>= sq
->size
)) {
6938 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sqtail
,
6939 "submission queue doorbell write value"
6940 " beyond queue size, sqid=%"PRIu32
","
6941 " new_tail=%"PRIu16
", ignoring",
6944 if (n
->outstanding_aers
) {
6945 nvme_enqueue_event(n
, NVME_AER_TYPE_ERROR
,
6946 NVME_AER_INFO_ERR_INVALID_DB_VALUE
,
6947 NVME_LOG_ERROR_INFO
);
6953 trace_pci_nvme_mmio_doorbell_sq(sq
->sqid
, new_tail
);
6955 sq
->tail
= new_tail
;
6956 if (!qid
&& n
->dbbuf_enabled
) {
6958 * The spec states "the host shall also update the controller's
6959 * corresponding doorbell property to match the value of that entry
6960 * in the Shadow Doorbell buffer."
6962 * Since this context is currently a VM trap, we can safely enforce
6963 * the requirement from the device side in case the host is
6966 * Note, we shouldn't have to do this, but various drivers
6967 * including ones that run on Linux, are not updating Admin Queues,
6968 * so we can't trust reading it for an appropriate sq tail.
6970 pci_dma_write(pci
, sq
->db_addr
, &sq
->tail
, sizeof(sq
->tail
));
6973 qemu_bh_schedule(sq
->bh
);
6977 static void nvme_mmio_write(void *opaque
, hwaddr addr
, uint64_t data
,
6980 NvmeCtrl
*n
= (NvmeCtrl
*)opaque
;
6982 trace_pci_nvme_mmio_write(addr
, data
, size
);
6984 if (pci_is_vf(PCI_DEVICE(n
)) && !nvme_sctrl(n
)->scs
&&
6985 addr
!= NVME_REG_CSTS
) {
6986 trace_pci_nvme_err_ignored_mmio_vf_offline(addr
, size
);
6990 if (addr
< sizeof(n
->bar
)) {
6991 nvme_write_bar(n
, addr
, data
, size
);
6993 nvme_process_db(n
, addr
, data
);
6997 static const MemoryRegionOps nvme_mmio_ops
= {
6998 .read
= nvme_mmio_read
,
6999 .write
= nvme_mmio_write
,
7000 .endianness
= DEVICE_LITTLE_ENDIAN
,
7002 .min_access_size
= 2,
7003 .max_access_size
= 8,
7007 static void nvme_cmb_write(void *opaque
, hwaddr addr
, uint64_t data
,
7010 NvmeCtrl
*n
= (NvmeCtrl
*)opaque
;
7011 stn_le_p(&n
->cmb
.buf
[addr
], size
, data
);
7014 static uint64_t nvme_cmb_read(void *opaque
, hwaddr addr
, unsigned size
)
7016 NvmeCtrl
*n
= (NvmeCtrl
*)opaque
;
7017 return ldn_le_p(&n
->cmb
.buf
[addr
], size
);
7020 static const MemoryRegionOps nvme_cmb_ops
= {
7021 .read
= nvme_cmb_read
,
7022 .write
= nvme_cmb_write
,
7023 .endianness
= DEVICE_LITTLE_ENDIAN
,
7025 .min_access_size
= 1,
7026 .max_access_size
= 8,
7030 static bool nvme_check_params(NvmeCtrl
*n
, Error
**errp
)
7032 NvmeParams
*params
= &n
->params
;
7034 if (params
->num_queues
) {
7035 warn_report("num_queues is deprecated; please use max_ioqpairs "
7038 params
->max_ioqpairs
= params
->num_queues
- 1;
7041 if (n
->namespace.blkconf
.blk
&& n
->subsys
) {
7042 error_setg(errp
, "subsystem support is unavailable with legacy "
7043 "namespace ('drive' property)");
7047 if (params
->max_ioqpairs
< 1 ||
7048 params
->max_ioqpairs
> NVME_MAX_IOQPAIRS
) {
7049 error_setg(errp
, "max_ioqpairs must be between 1 and %d",
7054 if (params
->msix_qsize
< 1 ||
7055 params
->msix_qsize
> PCI_MSIX_FLAGS_QSIZE
+ 1) {
7056 error_setg(errp
, "msix_qsize must be between 1 and %d",
7057 PCI_MSIX_FLAGS_QSIZE
+ 1);
7061 if (!params
->serial
) {
7062 error_setg(errp
, "serial property not set");
7067 if (host_memory_backend_is_mapped(n
->pmr
.dev
)) {
7068 error_setg(errp
, "can't use already busy memdev: %s",
7069 object_get_canonical_path_component(OBJECT(n
->pmr
.dev
)));
7073 if (!is_power_of_2(n
->pmr
.dev
->size
)) {
7074 error_setg(errp
, "pmr backend size needs to be power of 2 in size");
7078 host_memory_backend_set_mapped(n
->pmr
.dev
, true);
7081 if (n
->params
.zasl
> n
->params
.mdts
) {
7082 error_setg(errp
, "zoned.zasl (Zone Append Size Limit) must be less "
7083 "than or equal to mdts (Maximum Data Transfer Size)");
7087 if (!n
->params
.vsl
) {
7088 error_setg(errp
, "vsl must be non-zero");
7092 if (params
->sriov_max_vfs
) {
7094 error_setg(errp
, "subsystem is required for the use of SR-IOV");
7098 if (params
->sriov_max_vfs
> NVME_MAX_VFS
) {
7099 error_setg(errp
, "sriov_max_vfs must be between 0 and %d",
7104 if (params
->cmb_size_mb
) {
7105 error_setg(errp
, "CMB is not supported with SR-IOV");
7110 error_setg(errp
, "PMR is not supported with SR-IOV");
7114 if (!params
->sriov_vq_flexible
|| !params
->sriov_vi_flexible
) {
7115 error_setg(errp
, "both sriov_vq_flexible and sriov_vi_flexible"
7116 " must be set for the use of SR-IOV");
7120 if (params
->sriov_vq_flexible
< params
->sriov_max_vfs
* 2) {
7121 error_setg(errp
, "sriov_vq_flexible must be greater than or equal"
7122 " to %d (sriov_max_vfs * 2)", params
->sriov_max_vfs
* 2);
7126 if (params
->max_ioqpairs
< params
->sriov_vq_flexible
+ 2) {
7127 error_setg(errp
, "(max_ioqpairs - sriov_vq_flexible) must be"
7128 " greater than or equal to 2");
7132 if (params
->sriov_vi_flexible
< params
->sriov_max_vfs
) {
7133 error_setg(errp
, "sriov_vi_flexible must be greater than or equal"
7134 " to %d (sriov_max_vfs)", params
->sriov_max_vfs
);
7138 if (params
->msix_qsize
< params
->sriov_vi_flexible
+ 1) {
7139 error_setg(errp
, "(msix_qsize - sriov_vi_flexible) must be"
7140 " greater than or equal to 1");
7144 if (params
->sriov_max_vi_per_vf
&&
7145 (params
->sriov_max_vi_per_vf
- 1) % NVME_VF_RES_GRANULARITY
) {
7146 error_setg(errp
, "sriov_max_vi_per_vf must meet:"
7147 " (sriov_max_vi_per_vf - 1) %% %d == 0 and"
7148 " sriov_max_vi_per_vf >= 1", NVME_VF_RES_GRANULARITY
);
7152 if (params
->sriov_max_vq_per_vf
&&
7153 (params
->sriov_max_vq_per_vf
< 2 ||
7154 (params
->sriov_max_vq_per_vf
- 1) % NVME_VF_RES_GRANULARITY
)) {
7155 error_setg(errp
, "sriov_max_vq_per_vf must meet:"
7156 " (sriov_max_vq_per_vf - 1) %% %d == 0 and"
7157 " sriov_max_vq_per_vf >= 2", NVME_VF_RES_GRANULARITY
);
7165 static void nvme_init_state(NvmeCtrl
*n
)
7167 NvmePriCtrlCap
*cap
= &n
->pri_ctrl_cap
;
7168 NvmeSecCtrlList
*list
= &n
->sec_ctrl_list
;
7169 NvmeSecCtrlEntry
*sctrl
;
7170 PCIDevice
*pci
= PCI_DEVICE(n
);
7174 if (pci_is_vf(pci
)) {
7175 sctrl
= nvme_sctrl(n
);
7177 n
->conf_ioqpairs
= sctrl
->nvq
? le16_to_cpu(sctrl
->nvq
) - 1 : 0;
7178 n
->conf_msix_qsize
= sctrl
->nvi
? le16_to_cpu(sctrl
->nvi
) : 1;
7180 max_vfs
= n
->params
.sriov_max_vfs
;
7181 n
->conf_ioqpairs
= n
->params
.max_ioqpairs
;
7182 n
->conf_msix_qsize
= n
->params
.msix_qsize
;
7185 n
->sq
= g_new0(NvmeSQueue
*, n
->params
.max_ioqpairs
+ 1);
7186 n
->cq
= g_new0(NvmeCQueue
*, n
->params
.max_ioqpairs
+ 1);
7187 n
->temperature
= NVME_TEMPERATURE
;
7188 n
->features
.temp_thresh_hi
= NVME_TEMPERATURE_WARNING
;
7189 n
->starttime_ms
= qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
);
7190 n
->aer_reqs
= g_new0(NvmeRequest
*, n
->params
.aerl
+ 1);
7191 QTAILQ_INIT(&n
->aer_queue
);
7193 list
->numcntl
= cpu_to_le16(max_vfs
);
7194 for (i
= 0; i
< max_vfs
; i
++) {
7195 sctrl
= &list
->sec
[i
];
7196 sctrl
->pcid
= cpu_to_le16(n
->cntlid
);
7197 sctrl
->vfn
= cpu_to_le16(i
+ 1);
7200 cap
->cntlid
= cpu_to_le16(n
->cntlid
);
7201 cap
->crt
= NVME_CRT_VQ
| NVME_CRT_VI
;
7203 if (pci_is_vf(pci
)) {
7204 cap
->vqprt
= cpu_to_le16(1 + n
->conf_ioqpairs
);
7206 cap
->vqprt
= cpu_to_le16(1 + n
->params
.max_ioqpairs
-
7207 n
->params
.sriov_vq_flexible
);
7208 cap
->vqfrt
= cpu_to_le32(n
->params
.sriov_vq_flexible
);
7209 cap
->vqrfap
= cap
->vqfrt
;
7210 cap
->vqgran
= cpu_to_le16(NVME_VF_RES_GRANULARITY
);
7211 cap
->vqfrsm
= n
->params
.sriov_max_vq_per_vf
?
7212 cpu_to_le16(n
->params
.sriov_max_vq_per_vf
) :
7213 cap
->vqfrt
/ MAX(max_vfs
, 1);
7216 if (pci_is_vf(pci
)) {
7217 cap
->viprt
= cpu_to_le16(n
->conf_msix_qsize
);
7219 cap
->viprt
= cpu_to_le16(n
->params
.msix_qsize
-
7220 n
->params
.sriov_vi_flexible
);
7221 cap
->vifrt
= cpu_to_le32(n
->params
.sriov_vi_flexible
);
7222 cap
->virfap
= cap
->vifrt
;
7223 cap
->vigran
= cpu_to_le16(NVME_VF_RES_GRANULARITY
);
7224 cap
->vifrsm
= n
->params
.sriov_max_vi_per_vf
?
7225 cpu_to_le16(n
->params
.sriov_max_vi_per_vf
) :
7226 cap
->vifrt
/ MAX(max_vfs
, 1);
7230 static void nvme_init_cmb(NvmeCtrl
*n
, PCIDevice
*pci_dev
)
7232 uint64_t cmb_size
= n
->params
.cmb_size_mb
* MiB
;
7233 uint64_t cap
= ldq_le_p(&n
->bar
.cap
);
7235 n
->cmb
.buf
= g_malloc0(cmb_size
);
7236 memory_region_init_io(&n
->cmb
.mem
, OBJECT(n
), &nvme_cmb_ops
, n
,
7237 "nvme-cmb", cmb_size
);
7238 pci_register_bar(pci_dev
, NVME_CMB_BIR
,
7239 PCI_BASE_ADDRESS_SPACE_MEMORY
|
7240 PCI_BASE_ADDRESS_MEM_TYPE_64
|
7241 PCI_BASE_ADDRESS_MEM_PREFETCH
, &n
->cmb
.mem
);
7243 NVME_CAP_SET_CMBS(cap
, 1);
7244 stq_le_p(&n
->bar
.cap
, cap
);
7246 if (n
->params
.legacy_cmb
) {
7247 nvme_cmb_enable_regs(n
);
7252 static void nvme_init_pmr(NvmeCtrl
*n
, PCIDevice
*pci_dev
)
7254 uint32_t pmrcap
= ldl_le_p(&n
->bar
.pmrcap
);
7256 NVME_PMRCAP_SET_RDS(pmrcap
, 1);
7257 NVME_PMRCAP_SET_WDS(pmrcap
, 1);
7258 NVME_PMRCAP_SET_BIR(pmrcap
, NVME_PMR_BIR
);
7259 /* Turn on bit 1 support */
7260 NVME_PMRCAP_SET_PMRWBM(pmrcap
, 0x02);
7261 NVME_PMRCAP_SET_CMSS(pmrcap
, 1);
7262 stl_le_p(&n
->bar
.pmrcap
, pmrcap
);
7264 pci_register_bar(pci_dev
, NVME_PMR_BIR
,
7265 PCI_BASE_ADDRESS_SPACE_MEMORY
|
7266 PCI_BASE_ADDRESS_MEM_TYPE_64
|
7267 PCI_BASE_ADDRESS_MEM_PREFETCH
, &n
->pmr
.dev
->mr
);
7269 memory_region_set_enabled(&n
->pmr
.dev
->mr
, false);
7272 static uint64_t nvme_bar_size(unsigned total_queues
, unsigned total_irqs
,
7273 unsigned *msix_table_offset
,
7274 unsigned *msix_pba_offset
)
7276 uint64_t bar_size
, msix_table_size
, msix_pba_size
;
7278 bar_size
= sizeof(NvmeBar
) + 2 * total_queues
* NVME_DB_SIZE
;
7279 bar_size
= QEMU_ALIGN_UP(bar_size
, 4 * KiB
);
7281 if (msix_table_offset
) {
7282 *msix_table_offset
= bar_size
;
7285 msix_table_size
= PCI_MSIX_ENTRY_SIZE
* total_irqs
;
7286 bar_size
+= msix_table_size
;
7287 bar_size
= QEMU_ALIGN_UP(bar_size
, 4 * KiB
);
7289 if (msix_pba_offset
) {
7290 *msix_pba_offset
= bar_size
;
7293 msix_pba_size
= QEMU_ALIGN_UP(total_irqs
, 64) / 8;
7294 bar_size
+= msix_pba_size
;
7296 bar_size
= pow2ceil(bar_size
);
7300 static void nvme_init_sriov(NvmeCtrl
*n
, PCIDevice
*pci_dev
, uint16_t offset
)
7302 uint16_t vf_dev_id
= n
->params
.use_intel_id
?
7303 PCI_DEVICE_ID_INTEL_NVME
: PCI_DEVICE_ID_REDHAT_NVME
;
7304 NvmePriCtrlCap
*cap
= &n
->pri_ctrl_cap
;
7305 uint64_t bar_size
= nvme_bar_size(le16_to_cpu(cap
->vqfrsm
),
7306 le16_to_cpu(cap
->vifrsm
),
7309 pcie_sriov_pf_init(pci_dev
, offset
, "nvme", vf_dev_id
,
7310 n
->params
.sriov_max_vfs
, n
->params
.sriov_max_vfs
,
7311 NVME_VF_OFFSET
, NVME_VF_STRIDE
);
7313 pcie_sriov_pf_init_vf_bar(pci_dev
, 0, PCI_BASE_ADDRESS_SPACE_MEMORY
|
7314 PCI_BASE_ADDRESS_MEM_TYPE_64
, bar_size
);
7317 static int nvme_add_pm_capability(PCIDevice
*pci_dev
, uint8_t offset
)
7322 ret
= pci_add_capability(pci_dev
, PCI_CAP_ID_PM
, offset
,
7323 PCI_PM_SIZEOF
, &err
);
7325 error_report_err(err
);
7329 pci_set_word(pci_dev
->config
+ offset
+ PCI_PM_PMC
,
7330 PCI_PM_CAP_VER_1_2
);
7331 pci_set_word(pci_dev
->config
+ offset
+ PCI_PM_CTRL
,
7332 PCI_PM_CTRL_NO_SOFT_RESET
);
7333 pci_set_word(pci_dev
->wmask
+ offset
+ PCI_PM_CTRL
,
7334 PCI_PM_CTRL_STATE_MASK
);
7339 static bool nvme_init_pci(NvmeCtrl
*n
, PCIDevice
*pci_dev
, Error
**errp
)
7342 uint8_t *pci_conf
= pci_dev
->config
;
7344 unsigned msix_table_offset
, msix_pba_offset
;
7347 pci_conf
[PCI_INTERRUPT_PIN
] = 1;
7348 pci_config_set_prog_interface(pci_conf
, 0x2);
7350 if (n
->params
.use_intel_id
) {
7351 pci_config_set_vendor_id(pci_conf
, PCI_VENDOR_ID_INTEL
);
7352 pci_config_set_device_id(pci_conf
, PCI_DEVICE_ID_INTEL_NVME
);
7354 pci_config_set_vendor_id(pci_conf
, PCI_VENDOR_ID_REDHAT
);
7355 pci_config_set_device_id(pci_conf
, PCI_DEVICE_ID_REDHAT_NVME
);
7358 pci_config_set_class(pci_conf
, PCI_CLASS_STORAGE_EXPRESS
);
7359 nvme_add_pm_capability(pci_dev
, 0x60);
7360 pcie_endpoint_cap_init(pci_dev
, 0x80);
7361 pcie_cap_flr_init(pci_dev
);
7362 if (n
->params
.sriov_max_vfs
) {
7363 pcie_ari_init(pci_dev
, 0x100, 1);
7366 /* add one to max_ioqpairs to account for the admin queue pair */
7367 bar_size
= nvme_bar_size(n
->params
.max_ioqpairs
+ 1, n
->params
.msix_qsize
,
7368 &msix_table_offset
, &msix_pba_offset
);
7370 memory_region_init(&n
->bar0
, OBJECT(n
), "nvme-bar0", bar_size
);
7371 memory_region_init_io(&n
->iomem
, OBJECT(n
), &nvme_mmio_ops
, n
, "nvme",
7373 memory_region_add_subregion(&n
->bar0
, 0, &n
->iomem
);
7375 if (pci_is_vf(pci_dev
)) {
7376 pcie_sriov_vf_register_bar(pci_dev
, 0, &n
->bar0
);
7378 pci_register_bar(pci_dev
, 0, PCI_BASE_ADDRESS_SPACE_MEMORY
|
7379 PCI_BASE_ADDRESS_MEM_TYPE_64
, &n
->bar0
);
7381 ret
= msix_init(pci_dev
, n
->params
.msix_qsize
,
7382 &n
->bar0
, 0, msix_table_offset
,
7383 &n
->bar0
, 0, msix_pba_offset
, 0, errp
);
7384 if (ret
== -ENOTSUP
) {
7385 /* report that msix is not supported, but do not error out */
7386 warn_report_err(*errp
);
7388 } else if (ret
< 0) {
7389 /* propagate error to caller */
7393 nvme_update_msixcap_ts(pci_dev
, n
->conf_msix_qsize
);
7395 if (n
->params
.cmb_size_mb
) {
7396 nvme_init_cmb(n
, pci_dev
);
7400 nvme_init_pmr(n
, pci_dev
);
7403 if (!pci_is_vf(pci_dev
) && n
->params
.sriov_max_vfs
) {
7404 nvme_init_sriov(n
, pci_dev
, 0x120);
7410 static void nvme_init_subnqn(NvmeCtrl
*n
)
7412 NvmeSubsystem
*subsys
= n
->subsys
;
7413 NvmeIdCtrl
*id
= &n
->id_ctrl
;
7416 snprintf((char *)id
->subnqn
, sizeof(id
->subnqn
),
7417 "nqn.2019-08.org.qemu:%s", n
->params
.serial
);
7419 pstrcpy((char *)id
->subnqn
, sizeof(id
->subnqn
), (char*)subsys
->subnqn
);
7423 static void nvme_init_ctrl(NvmeCtrl
*n
, PCIDevice
*pci_dev
)
7425 NvmeIdCtrl
*id
= &n
->id_ctrl
;
7426 uint8_t *pci_conf
= pci_dev
->config
;
7427 uint64_t cap
= ldq_le_p(&n
->bar
.cap
);
7428 NvmeSecCtrlEntry
*sctrl
= nvme_sctrl(n
);
7431 id
->vid
= cpu_to_le16(pci_get_word(pci_conf
+ PCI_VENDOR_ID
));
7432 id
->ssvid
= cpu_to_le16(pci_get_word(pci_conf
+ PCI_SUBSYSTEM_VENDOR_ID
));
7433 strpadcpy((char *)id
->mn
, sizeof(id
->mn
), "QEMU NVMe Ctrl", ' ');
7434 strpadcpy((char *)id
->fr
, sizeof(id
->fr
), QEMU_VERSION
, ' ');
7435 strpadcpy((char *)id
->sn
, sizeof(id
->sn
), n
->params
.serial
, ' ');
7437 id
->cntlid
= cpu_to_le16(n
->cntlid
);
7439 id
->oaes
= cpu_to_le32(NVME_OAES_NS_ATTR
);
7440 ctratt
= NVME_CTRATT_ELBAS
;
7444 if (n
->params
.use_intel_id
) {
7454 id
->mdts
= n
->params
.mdts
;
7455 id
->ver
= cpu_to_le32(NVME_SPEC_VER
);
7457 cpu_to_le16(NVME_OACS_NS_MGMT
| NVME_OACS_FORMAT
| NVME_OACS_DBBUF
);
7458 id
->cntrltype
= 0x1;
7461 * Because the controller always completes the Abort command immediately,
7462 * there can never be more than one concurrently executing Abort command,
7463 * so this value is never used for anything. Note that there can easily be
7464 * many Abort commands in the queues, but they are not considered
7465 * "executing" until processed by nvme_abort.
7467 * The specification recommends a value of 3 for Abort Command Limit (four
7468 * concurrently outstanding Abort commands), so lets use that though it is
7472 id
->aerl
= n
->params
.aerl
;
7473 id
->frmw
= (NVME_NUM_FW_SLOTS
<< 1) | NVME_FRMW_SLOT1_RO
;
7474 id
->lpa
= NVME_LPA_NS_SMART
| NVME_LPA_CSE
| NVME_LPA_EXTENDED
;
7476 /* recommended default value (~70 C) */
7477 id
->wctemp
= cpu_to_le16(NVME_TEMPERATURE_WARNING
);
7478 id
->cctemp
= cpu_to_le16(NVME_TEMPERATURE_CRITICAL
);
7480 id
->sqes
= (0x6 << 4) | 0x6;
7481 id
->cqes
= (0x4 << 4) | 0x4;
7482 id
->nn
= cpu_to_le32(NVME_MAX_NAMESPACES
);
7483 id
->oncs
= cpu_to_le16(NVME_ONCS_WRITE_ZEROES
| NVME_ONCS_TIMESTAMP
|
7484 NVME_ONCS_FEATURES
| NVME_ONCS_DSM
|
7485 NVME_ONCS_COMPARE
| NVME_ONCS_COPY
);
7488 * NOTE: If this device ever supports a command set that does NOT use 0x0
7489 * as a Flush-equivalent operation, support for the broadcast NSID in Flush
7490 * should probably be removed.
7492 * See comment in nvme_io_cmd.
7494 id
->vwc
= NVME_VWC_NSID_BROADCAST_SUPPORT
| NVME_VWC_PRESENT
;
7496 id
->ocfs
= cpu_to_le16(NVME_OCFS_COPY_FORMAT_0
| NVME_OCFS_COPY_FORMAT_1
);
7497 id
->sgls
= cpu_to_le32(NVME_CTRL_SGLS_SUPPORT_NO_ALIGN
);
7499 nvme_init_subnqn(n
);
7501 id
->psd
[0].mp
= cpu_to_le16(0x9c4);
7502 id
->psd
[0].enlat
= cpu_to_le32(0x10);
7503 id
->psd
[0].exlat
= cpu_to_le32(0x4);
7506 id
->cmic
|= NVME_CMIC_MULTI_CTRL
;
7507 ctratt
|= NVME_CTRATT_ENDGRPS
;
7509 id
->endgidmax
= cpu_to_le16(0x1);
7512 id
->ctratt
= cpu_to_le32(ctratt
);
7514 NVME_CAP_SET_MQES(cap
, 0x7ff);
7515 NVME_CAP_SET_CQR(cap
, 1);
7516 NVME_CAP_SET_TO(cap
, 0xf);
7517 NVME_CAP_SET_CSS(cap
, NVME_CAP_CSS_NVM
);
7518 NVME_CAP_SET_CSS(cap
, NVME_CAP_CSS_CSI_SUPP
);
7519 NVME_CAP_SET_CSS(cap
, NVME_CAP_CSS_ADMIN_ONLY
);
7520 NVME_CAP_SET_MPSMAX(cap
, 4);
7521 NVME_CAP_SET_CMBS(cap
, n
->params
.cmb_size_mb
? 1 : 0);
7522 NVME_CAP_SET_PMRS(cap
, n
->pmr
.dev
? 1 : 0);
7523 stq_le_p(&n
->bar
.cap
, cap
);
7525 stl_le_p(&n
->bar
.vs
, NVME_SPEC_VER
);
7526 n
->bar
.intmc
= n
->bar
.intms
= 0;
7528 if (pci_is_vf(pci_dev
) && !sctrl
->scs
) {
7529 stl_le_p(&n
->bar
.csts
, NVME_CSTS_FAILED
);
7533 static int nvme_init_subsys(NvmeCtrl
*n
, Error
**errp
)
7541 cntlid
= nvme_subsys_register_ctrl(n
, errp
);
7551 void nvme_attach_ns(NvmeCtrl
*n
, NvmeNamespace
*ns
)
7553 uint32_t nsid
= ns
->params
.nsid
;
7554 assert(nsid
&& nsid
<= NVME_MAX_NAMESPACES
);
7556 n
->namespaces
[nsid
] = ns
;
7559 n
->dmrsl
= MIN_NON_ZERO(n
->dmrsl
,
7560 BDRV_REQUEST_MAX_BYTES
/ nvme_l2b(ns
, 1));
7563 static void nvme_realize(PCIDevice
*pci_dev
, Error
**errp
)
7565 NvmeCtrl
*n
= NVME(pci_dev
);
7566 DeviceState
*dev
= DEVICE(pci_dev
);
7568 NvmeCtrl
*pn
= NVME(pcie_sriov_get_pf(pci_dev
));
7570 if (pci_is_vf(pci_dev
)) {
7572 * VFs derive settings from the parent. PF's lifespan exceeds
7573 * that of VF's, so it's safe to share params.serial.
7575 memcpy(&n
->params
, &pn
->params
, sizeof(NvmeParams
));
7576 n
->subsys
= pn
->subsys
;
7579 if (!nvme_check_params(n
, errp
)) {
7583 qbus_init(&n
->bus
, sizeof(NvmeBus
), TYPE_NVME_BUS
, dev
, dev
->id
);
7585 if (nvme_init_subsys(n
, errp
)) {
7589 if (!nvme_init_pci(n
, pci_dev
, errp
)) {
7592 nvme_init_ctrl(n
, pci_dev
);
7594 /* setup a namespace if the controller drive property was given */
7595 if (n
->namespace.blkconf
.blk
) {
7597 ns
->params
.nsid
= 1;
7599 if (nvme_ns_setup(ns
, errp
)) {
7603 nvme_attach_ns(n
, ns
);
7607 static void nvme_exit(PCIDevice
*pci_dev
)
7609 NvmeCtrl
*n
= NVME(pci_dev
);
7613 nvme_ctrl_reset(n
, NVME_RESET_FUNCTION
);
7616 for (i
= 1; i
<= NVME_MAX_NAMESPACES
; i
++) {
7623 nvme_subsys_unregister_ctrl(n
->subsys
, n
);
7628 g_free(n
->aer_reqs
);
7630 if (n
->params
.cmb_size_mb
) {
7635 host_memory_backend_set_mapped(n
->pmr
.dev
, false);
7638 if (!pci_is_vf(pci_dev
) && n
->params
.sriov_max_vfs
) {
7639 pcie_sriov_pf_exit(pci_dev
);
7642 msix_uninit(pci_dev
, &n
->bar0
, &n
->bar0
);
7643 memory_region_del_subregion(&n
->bar0
, &n
->iomem
);
7646 static Property nvme_props
[] = {
7647 DEFINE_BLOCK_PROPERTIES(NvmeCtrl
, namespace.blkconf
),
7648 DEFINE_PROP_LINK("pmrdev", NvmeCtrl
, pmr
.dev
, TYPE_MEMORY_BACKEND
,
7649 HostMemoryBackend
*),
7650 DEFINE_PROP_LINK("subsys", NvmeCtrl
, subsys
, TYPE_NVME_SUBSYS
,
7652 DEFINE_PROP_STRING("serial", NvmeCtrl
, params
.serial
),
7653 DEFINE_PROP_UINT32("cmb_size_mb", NvmeCtrl
, params
.cmb_size_mb
, 0),
7654 DEFINE_PROP_UINT32("num_queues", NvmeCtrl
, params
.num_queues
, 0),
7655 DEFINE_PROP_UINT32("max_ioqpairs", NvmeCtrl
, params
.max_ioqpairs
, 64),
7656 DEFINE_PROP_UINT16("msix_qsize", NvmeCtrl
, params
.msix_qsize
, 65),
7657 DEFINE_PROP_UINT8("aerl", NvmeCtrl
, params
.aerl
, 3),
7658 DEFINE_PROP_UINT32("aer_max_queued", NvmeCtrl
, params
.aer_max_queued
, 64),
7659 DEFINE_PROP_UINT8("mdts", NvmeCtrl
, params
.mdts
, 7),
7660 DEFINE_PROP_UINT8("vsl", NvmeCtrl
, params
.vsl
, 7),
7661 DEFINE_PROP_BOOL("use-intel-id", NvmeCtrl
, params
.use_intel_id
, false),
7662 DEFINE_PROP_BOOL("legacy-cmb", NvmeCtrl
, params
.legacy_cmb
, false),
7663 DEFINE_PROP_BOOL("ioeventfd", NvmeCtrl
, params
.ioeventfd
, false),
7664 DEFINE_PROP_UINT8("zoned.zasl", NvmeCtrl
, params
.zasl
, 0),
7665 DEFINE_PROP_BOOL("zoned.auto_transition", NvmeCtrl
,
7666 params
.auto_transition_zones
, true),
7667 DEFINE_PROP_UINT8("sriov_max_vfs", NvmeCtrl
, params
.sriov_max_vfs
, 0),
7668 DEFINE_PROP_UINT16("sriov_vq_flexible", NvmeCtrl
,
7669 params
.sriov_vq_flexible
, 0),
7670 DEFINE_PROP_UINT16("sriov_vi_flexible", NvmeCtrl
,
7671 params
.sriov_vi_flexible
, 0),
7672 DEFINE_PROP_UINT8("sriov_max_vi_per_vf", NvmeCtrl
,
7673 params
.sriov_max_vi_per_vf
, 0),
7674 DEFINE_PROP_UINT8("sriov_max_vq_per_vf", NvmeCtrl
,
7675 params
.sriov_max_vq_per_vf
, 0),
7676 DEFINE_PROP_END_OF_LIST(),
7679 static void nvme_get_smart_warning(Object
*obj
, Visitor
*v
, const char *name
,
7680 void *opaque
, Error
**errp
)
7682 NvmeCtrl
*n
= NVME(obj
);
7683 uint8_t value
= n
->smart_critical_warning
;
7685 visit_type_uint8(v
, name
, &value
, errp
);
7688 static void nvme_set_smart_warning(Object
*obj
, Visitor
*v
, const char *name
,
7689 void *opaque
, Error
**errp
)
7691 NvmeCtrl
*n
= NVME(obj
);
7692 uint8_t value
, old_value
, cap
= 0, index
, event
;
7694 if (!visit_type_uint8(v
, name
, &value
, errp
)) {
7698 cap
= NVME_SMART_SPARE
| NVME_SMART_TEMPERATURE
| NVME_SMART_RELIABILITY
7699 | NVME_SMART_MEDIA_READ_ONLY
| NVME_SMART_FAILED_VOLATILE_MEDIA
;
7700 if (NVME_CAP_PMRS(ldq_le_p(&n
->bar
.cap
))) {
7701 cap
|= NVME_SMART_PMR_UNRELIABLE
;
7704 if ((value
& cap
) != value
) {
7705 error_setg(errp
, "unsupported smart critical warning bits: 0x%x",
7710 old_value
= n
->smart_critical_warning
;
7711 n
->smart_critical_warning
= value
;
7713 /* only inject new bits of smart critical warning */
7714 for (index
= 0; index
< NVME_SMART_WARN_MAX
; index
++) {
7716 if (value
& ~old_value
& event
)
7717 nvme_smart_event(n
, event
);
7721 static void nvme_pci_reset(DeviceState
*qdev
)
7723 PCIDevice
*pci_dev
= PCI_DEVICE(qdev
);
7724 NvmeCtrl
*n
= NVME(pci_dev
);
7726 trace_pci_nvme_pci_reset();
7727 nvme_ctrl_reset(n
, NVME_RESET_FUNCTION
);
7730 static void nvme_sriov_pre_write_ctrl(PCIDevice
*dev
, uint32_t address
,
7731 uint32_t val
, int len
)
7733 NvmeCtrl
*n
= NVME(dev
);
7734 NvmeSecCtrlEntry
*sctrl
;
7735 uint16_t sriov_cap
= dev
->exp
.sriov_cap
;
7736 uint32_t off
= address
- sriov_cap
;
7743 if (range_covers_byte(off
, len
, PCI_SRIOV_CTRL
)) {
7744 if (!(val
& PCI_SRIOV_CTRL_VFE
)) {
7745 num_vfs
= pci_get_word(dev
->config
+ sriov_cap
+ PCI_SRIOV_NUM_VF
);
7746 for (i
= 0; i
< num_vfs
; i
++) {
7747 sctrl
= &n
->sec_ctrl_list
.sec
[i
];
7748 nvme_virt_set_state(n
, le16_to_cpu(sctrl
->scid
), false);
7754 static void nvme_pci_write_config(PCIDevice
*dev
, uint32_t address
,
7755 uint32_t val
, int len
)
7757 nvme_sriov_pre_write_ctrl(dev
, address
, val
, len
);
7758 pci_default_write_config(dev
, address
, val
, len
);
7759 pcie_cap_flr_write_config(dev
, address
, val
, len
);
7762 static const VMStateDescription nvme_vmstate
= {
7767 static void nvme_class_init(ObjectClass
*oc
, void *data
)
7769 DeviceClass
*dc
= DEVICE_CLASS(oc
);
7770 PCIDeviceClass
*pc
= PCI_DEVICE_CLASS(oc
);
7772 pc
->realize
= nvme_realize
;
7773 pc
->config_write
= nvme_pci_write_config
;
7774 pc
->exit
= nvme_exit
;
7775 pc
->class_id
= PCI_CLASS_STORAGE_EXPRESS
;
7778 set_bit(DEVICE_CATEGORY_STORAGE
, dc
->categories
);
7779 dc
->desc
= "Non-Volatile Memory Express";
7780 device_class_set_props(dc
, nvme_props
);
7781 dc
->vmsd
= &nvme_vmstate
;
7782 dc
->reset
= nvme_pci_reset
;
7785 static void nvme_instance_init(Object
*obj
)
7787 NvmeCtrl
*n
= NVME(obj
);
7789 device_add_bootindex_property(obj
, &n
->namespace.blkconf
.bootindex
,
7790 "bootindex", "/namespace@1,0",
7793 object_property_add(obj
, "smart_critical_warning", "uint8",
7794 nvme_get_smart_warning
,
7795 nvme_set_smart_warning
, NULL
, NULL
);
7798 static const TypeInfo nvme_info
= {
7800 .parent
= TYPE_PCI_DEVICE
,
7801 .instance_size
= sizeof(NvmeCtrl
),
7802 .instance_init
= nvme_instance_init
,
7803 .class_init
= nvme_class_init
,
7804 .interfaces
= (InterfaceInfo
[]) {
7805 { INTERFACE_PCIE_DEVICE
},
7810 static const TypeInfo nvme_bus_info
= {
7811 .name
= TYPE_NVME_BUS
,
7813 .instance_size
= sizeof(NvmeBus
),
7816 static void nvme_register_types(void)
7818 type_register_static(&nvme_info
);
7819 type_register_static(&nvme_bus_info
);
7822 type_init(nvme_register_types
)