]> git.proxmox.com Git - mirror_qemu.git/blob - hw/nvme/ctrl.c
projects / mirror_qemu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
hw/nvme: consider COPY command in nvme_aio_err
[mirror_qemu.git] / hw / nvme / ctrl.c
1 /*
2 * QEMU NVM Express Controller
3 *
4 * Copyright (c) 2012, Intel Corporation
5 *
6 * Written by Keith Busch <keith.busch@intel.com>
7 *
8 * This code is licensed under the GNU GPL v2 or later.
9 */
10
11 /**
12 * Reference Specs: http://www.nvmexpress.org, 1.4, 1.3, 1.2, 1.1, 1.0e
13 *
14 * https://nvmexpress.org/developers/nvme-specification/
15 *
16 *
17 * Notes on coding style
18 * ---------------------
19 * While QEMU coding style prefers lowercase hexadecimals in constants, the
20 * NVMe subsystem use thes format from the NVMe specifications in the comments
21 * (i.e. 'h' suffix instead of '0x' prefix).
22 *
23 * Usage
24 * -----
25 * See docs/system/nvme.rst for extensive documentation.
26 *
27 * Add options:
28 * -drive file=<file>,if=none,id=<drive_id>
29 * -device nvme-subsys,id=<subsys_id>,nqn=<nqn_id>
30 * -device nvme,serial=<serial>,id=<bus_name>, \
31 * cmb_size_mb=<cmb_size_mb[optional]>, \
32 * [pmrdev=<mem_backend_file_id>,] \
33 * max_ioqpairs=<N[optional]>, \
34 * aerl=<N[optional]>,aer_max_queued=<N[optional]>, \
35 * mdts=<N[optional]>,vsl=<N[optional]>, \
36 * zoned.zasl=<N[optional]>, \
37 * zoned.auto_transition=<on|off[optional]>, \
38 * sriov_max_vfs=<N[optional]> \
39 * sriov_vq_flexible=<N[optional]> \
40 * sriov_vi_flexible=<N[optional]> \
41 * sriov_max_vi_per_vf=<N[optional]> \
42 * sriov_max_vq_per_vf=<N[optional]> \
43 * subsys=<subsys_id>
44 * -device nvme-ns,drive=<drive_id>,bus=<bus_name>,nsid=<nsid>,\
45 * zoned=<true|false[optional]>, \
46 * subsys=<subsys_id>,shared=<true|false[optional]>, \
47 * detached=<true|false[optional]>, \
48 * zoned.zone_size=<N[optional]>, \
49 * zoned.zone_capacity=<N[optional]>, \
50 * zoned.descr_ext_size=<N[optional]>, \
51 * zoned.max_active=<N[optional]>, \
52 * zoned.max_open=<N[optional]>, \
53 * zoned.cross_read=<true|false[optional]>
54 *
55 * Note cmb_size_mb denotes size of CMB in MB. CMB is assumed to be at
56 * offset 0 in BAR2 and supports only WDS, RDS and SQS for now. By default, the
57 * device will use the "v1.4 CMB scheme" - use the `legacy-cmb` parameter to
58 * always enable the CMBLOC and CMBSZ registers (v1.3 behavior).
59 *
60 * Enabling pmr emulation can be achieved by pointing to memory-backend-file.
61 * For example:
62 * -object memory-backend-file,id=<mem_id>,share=on,mem-path=<file_path>, \
63 * size=<size> .... -device nvme,...,pmrdev=<mem_id>
64 *
65 * The PMR will use BAR 4/5 exclusively.
66 *
67 * To place controller(s) and namespace(s) to a subsystem, then provide
68 * nvme-subsys device as above.
69 *
70 * nvme subsystem device parameters
71 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
72 * - `nqn`
73 * This parameter provides the `<nqn_id>` part of the string
74 * `nqn.2019-08.org.qemu:<nqn_id>` which will be reported in the SUBNQN field
75 * of subsystem controllers. Note that `<nqn_id>` should be unique per
76 * subsystem, but this is not enforced by QEMU. If not specified, it will
77 * default to the value of the `id` parameter (`<subsys_id>`).
78 *
79 * nvme device parameters
80 * ~~~~~~~~~~~~~~~~~~~~~~
81 * - `subsys`
82 * Specifying this parameter attaches the controller to the subsystem and
83 * the SUBNQN field in the controller will report the NQN of the subsystem
84 * device. This also enables multi controller capability represented in
85 * Identify Controller data structure in CMIC (Controller Multi-path I/O and
86 * Namespace Sharing Capabilities).
87 *
88 * - `aerl`
89 * The Asynchronous Event Request Limit (AERL). Indicates the maximum number
90 * of concurrently outstanding Asynchronous Event Request commands support
91 * by the controller. This is a 0's based value.
92 *
93 * - `aer_max_queued`
94 * This is the maximum number of events that the device will enqueue for
95 * completion when there are no outstanding AERs. When the maximum number of
96 * enqueued events are reached, subsequent events will be dropped.
97 *
98 * - `mdts`
99 * Indicates the maximum data transfer size for a command that transfers data
100 * between host-accessible memory and the controller. The value is specified
101 * as a power of two (2^n) and is in units of the minimum memory page size
102 * (CAP.MPSMIN). The default value is 7 (i.e. 512 KiB).
103 *
104 * - `vsl`
105 * Indicates the maximum data size limit for the Verify command. Like `mdts`,
106 * this value is specified as a power of two (2^n) and is in units of the
107 * minimum memory page size (CAP.MPSMIN). The default value is 7 (i.e. 512
108 * KiB).
109 *
110 * - `zoned.zasl`
111 * Indicates the maximum data transfer size for the Zone Append command. Like
112 * `mdts`, the value is specified as a power of two (2^n) and is in units of
113 * the minimum memory page size (CAP.MPSMIN). The default value is 0 (i.e.
114 * defaulting to the value of `mdts`).
115 *
116 * - `zoned.auto_transition`
117 * Indicates if zones in zone state implicitly opened can be automatically
118 * transitioned to zone state closed for resource management purposes.
119 * Defaults to 'on'.
120 *
121 * - `sriov_max_vfs`
122 * Indicates the maximum number of PCIe virtual functions supported
123 * by the controller. The default value is 0. Specifying a non-zero value
124 * enables reporting of both SR-IOV and ARI capabilities by the NVMe device.
125 * Virtual function controllers will not report SR-IOV capability.
126 *
127 * NOTE: Single Root I/O Virtualization support is experimental.
128 * All the related parameters may be subject to change.
129 *
130 * - `sriov_vq_flexible`
131 * Indicates the total number of flexible queue resources assignable to all
132 * the secondary controllers. Implicitly sets the number of primary
133 * controller's private resources to `(max_ioqpairs - sriov_vq_flexible)`.
134 *
135 * - `sriov_vi_flexible`
136 * Indicates the total number of flexible interrupt resources assignable to
137 * all the secondary controllers. Implicitly sets the number of primary
138 * controller's private resources to `(msix_qsize - sriov_vi_flexible)`.
139 *
140 * - `sriov_max_vi_per_vf`
141 * Indicates the maximum number of virtual interrupt resources assignable
142 * to a secondary controller. The default 0 resolves to
143 * `(sriov_vi_flexible / sriov_max_vfs)`.
144 *
145 * - `sriov_max_vq_per_vf`
146 * Indicates the maximum number of virtual queue resources assignable to
147 * a secondary controller. The default 0 resolves to
148 * `(sriov_vq_flexible / sriov_max_vfs)`.
149 *
150 * nvme namespace device parameters
151 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
152 * - `shared`
153 * When the parent nvme device (as defined explicitly by the 'bus' parameter
154 * or implicitly by the most recently defined NvmeBus) is linked to an
155 * nvme-subsys device, the namespace will be attached to all controllers in
156 * the subsystem. If set to 'off' (the default), the namespace will remain a
157 * private namespace and may only be attached to a single controller at a
158 * time.
159 *
160 * - `detached`
161 * This parameter is only valid together with the `subsys` parameter. If left
162 * at the default value (`false/off`), the namespace will be attached to all
163 * controllers in the NVMe subsystem at boot-up. If set to `true/on`, the
164 * namespace will be available in the subsystem but not attached to any
165 * controllers.
166 *
167 * Setting `zoned` to true selects Zoned Command Set at the namespace.
168 * In this case, the following namespace properties are available to configure
169 * zoned operation:
170 * zoned.zone_size=<zone size in bytes, default: 128MiB>
171 * The number may be followed by K, M, G as in kilo-, mega- or giga-.
172 *
173 * zoned.zone_capacity=<zone capacity in bytes, default: zone size>
174 * The value 0 (default) forces zone capacity to be the same as zone
175 * size. The value of this property may not exceed zone size.
176 *
177 * zoned.descr_ext_size=<zone descriptor extension size, default 0>
178 * This value needs to be specified in 64B units. If it is zero,
179 * namespace(s) will not support zone descriptor extensions.
180 *
181 * zoned.max_active=<Maximum Active Resources (zones), default: 0>
182 * The default value means there is no limit to the number of
183 * concurrently active zones.
184 *
185 * zoned.max_open=<Maximum Open Resources (zones), default: 0>
186 * The default value means there is no limit to the number of
187 * concurrently open zones.
188 *
189 * zoned.cross_read=<enable RAZB, default: false>
190 * Setting this property to true enables Read Across Zone Boundaries.
191 */
192
193 #include "qemu/osdep.h"
194 #include "qemu/cutils.h"
195 #include "qemu/error-report.h"
196 #include "qemu/log.h"
197 #include "qemu/units.h"
198 #include "qemu/range.h"
199 #include "qapi/error.h"
200 #include "qapi/visitor.h"
201 #include "sysemu/sysemu.h"
202 #include "sysemu/block-backend.h"
203 #include "sysemu/hostmem.h"
204 #include "hw/pci/msix.h"
205 #include "hw/pci/pcie_sriov.h"
206 #include "migration/vmstate.h"
207
208 #include "nvme.h"
209 #include "dif.h"
210 #include "trace.h"
211
212 #define NVME_MAX_IOQPAIRS 0xffff
213 #define NVME_DB_SIZE 4
214 #define NVME_SPEC_VER 0x00010400
215 #define NVME_CMB_BIR 2
216 #define NVME_PMR_BIR 4
217 #define NVME_TEMPERATURE 0x143
218 #define NVME_TEMPERATURE_WARNING 0x157
219 #define NVME_TEMPERATURE_CRITICAL 0x175
220 #define NVME_NUM_FW_SLOTS 1
221 #define NVME_DEFAULT_MAX_ZA_SIZE (128 * KiB)
222 #define NVME_MAX_VFS 127
223 #define NVME_VF_RES_GRANULARITY 1
224 #define NVME_VF_OFFSET 0x1
225 #define NVME_VF_STRIDE 1
226
227 #define NVME_GUEST_ERR(trace, fmt, ...) \
228 do { \
229 (trace_##trace)(__VA_ARGS__); \
230 qemu_log_mask(LOG_GUEST_ERROR, #trace \
231 " in %s: " fmt "\n", __func__, ## __VA_ARGS__); \
232 } while (0)
233
234 static const bool nvme_feature_support[NVME_FID_MAX] = {
235 [NVME_ARBITRATION] = true,
236 [NVME_POWER_MANAGEMENT] = true,
237 [NVME_TEMPERATURE_THRESHOLD] = true,
238 [NVME_ERROR_RECOVERY] = true,
239 [NVME_VOLATILE_WRITE_CACHE] = true,
240 [NVME_NUMBER_OF_QUEUES] = true,
241 [NVME_INTERRUPT_COALESCING] = true,
242 [NVME_INTERRUPT_VECTOR_CONF] = true,
243 [NVME_WRITE_ATOMICITY] = true,
244 [NVME_ASYNCHRONOUS_EVENT_CONF] = true,
245 [NVME_TIMESTAMP] = true,
246 [NVME_HOST_BEHAVIOR_SUPPORT] = true,
247 [NVME_COMMAND_SET_PROFILE] = true,
248 [NVME_FDP_MODE] = true,
249 [NVME_FDP_EVENTS] = true,
250 };
251
252 static const uint32_t nvme_feature_cap[NVME_FID_MAX] = {
253 [NVME_TEMPERATURE_THRESHOLD] = NVME_FEAT_CAP_CHANGE,
254 [NVME_ERROR_RECOVERY] = NVME_FEAT_CAP_CHANGE | NVME_FEAT_CAP_NS,
255 [NVME_VOLATILE_WRITE_CACHE] = NVME_FEAT_CAP_CHANGE,
256 [NVME_NUMBER_OF_QUEUES] = NVME_FEAT_CAP_CHANGE,
257 [NVME_ASYNCHRONOUS_EVENT_CONF] = NVME_FEAT_CAP_CHANGE,
258 [NVME_TIMESTAMP] = NVME_FEAT_CAP_CHANGE,
259 [NVME_HOST_BEHAVIOR_SUPPORT] = NVME_FEAT_CAP_CHANGE,
260 [NVME_COMMAND_SET_PROFILE] = NVME_FEAT_CAP_CHANGE,
261 [NVME_FDP_MODE] = NVME_FEAT_CAP_CHANGE,
262 [NVME_FDP_EVENTS] = NVME_FEAT_CAP_CHANGE | NVME_FEAT_CAP_NS,
263 };
264
265 static const uint32_t nvme_cse_acs[256] = {
266 [NVME_ADM_CMD_DELETE_SQ] = NVME_CMD_EFF_CSUPP,
267 [NVME_ADM_CMD_CREATE_SQ] = NVME_CMD_EFF_CSUPP,
268 [NVME_ADM_CMD_GET_LOG_PAGE] = NVME_CMD_EFF_CSUPP,
269 [NVME_ADM_CMD_DELETE_CQ] = NVME_CMD_EFF_CSUPP,
270 [NVME_ADM_CMD_CREATE_CQ] = NVME_CMD_EFF_CSUPP,
271 [NVME_ADM_CMD_IDENTIFY] = NVME_CMD_EFF_CSUPP,
272 [NVME_ADM_CMD_ABORT] = NVME_CMD_EFF_CSUPP,
273 [NVME_ADM_CMD_SET_FEATURES] = NVME_CMD_EFF_CSUPP,
274 [NVME_ADM_CMD_GET_FEATURES] = NVME_CMD_EFF_CSUPP,
275 [NVME_ADM_CMD_ASYNC_EV_REQ] = NVME_CMD_EFF_CSUPP,
276 [NVME_ADM_CMD_NS_ATTACHMENT] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_NIC,
277 [NVME_ADM_CMD_VIRT_MNGMT] = NVME_CMD_EFF_CSUPP,
278 [NVME_ADM_CMD_DBBUF_CONFIG] = NVME_CMD_EFF_CSUPP,
279 [NVME_ADM_CMD_FORMAT_NVM] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
280 [NVME_ADM_CMD_DIRECTIVE_RECV] = NVME_CMD_EFF_CSUPP,
281 [NVME_ADM_CMD_DIRECTIVE_SEND] = NVME_CMD_EFF_CSUPP,
282 };
283
284 static const uint32_t nvme_cse_iocs_none[256];
285
286 static const uint32_t nvme_cse_iocs_nvm[256] = {
287 [NVME_CMD_FLUSH] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
288 [NVME_CMD_WRITE_ZEROES] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
289 [NVME_CMD_WRITE] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
290 [NVME_CMD_READ] = NVME_CMD_EFF_CSUPP,
291 [NVME_CMD_DSM] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
292 [NVME_CMD_VERIFY] = NVME_CMD_EFF_CSUPP,
293 [NVME_CMD_COPY] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
294 [NVME_CMD_COMPARE] = NVME_CMD_EFF_CSUPP,
295 [NVME_CMD_IO_MGMT_RECV] = NVME_CMD_EFF_CSUPP,
296 [NVME_CMD_IO_MGMT_SEND] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
297 };
298
299 static const uint32_t nvme_cse_iocs_zoned[256] = {
300 [NVME_CMD_FLUSH] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
301 [NVME_CMD_WRITE_ZEROES] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
302 [NVME_CMD_WRITE] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
303 [NVME_CMD_READ] = NVME_CMD_EFF_CSUPP,
304 [NVME_CMD_DSM] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
305 [NVME_CMD_VERIFY] = NVME_CMD_EFF_CSUPP,
306 [NVME_CMD_COPY] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
307 [NVME_CMD_COMPARE] = NVME_CMD_EFF_CSUPP,
308 [NVME_CMD_ZONE_APPEND] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
309 [NVME_CMD_ZONE_MGMT_SEND] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
310 [NVME_CMD_ZONE_MGMT_RECV] = NVME_CMD_EFF_CSUPP,
311 };
312
313 static void nvme_process_sq(void *opaque);
314 static void nvme_ctrl_reset(NvmeCtrl *n, NvmeResetType rst);
315 static inline uint64_t nvme_get_timestamp(const NvmeCtrl *n);
316
317 static uint16_t nvme_sqid(NvmeRequest *req)
318 {
319 return le16_to_cpu(req->sq->sqid);
320 }
321
322 static inline uint16_t nvme_make_pid(NvmeNamespace *ns, uint16_t rg,
323 uint16_t ph)
324 {
325 uint16_t rgif = ns->endgrp->fdp.rgif;
326
327 if (!rgif) {
328 return ph;
329 }
330
331 return (rg << (16 - rgif)) | ph;
332 }
333
334 static inline bool nvme_ph_valid(NvmeNamespace *ns, uint16_t ph)
335 {
336 return ph < ns->fdp.nphs;
337 }
338
339 static inline bool nvme_rg_valid(NvmeEnduranceGroup *endgrp, uint16_t rg)
340 {
341 return rg < endgrp->fdp.nrg;
342 }
343
344 static inline uint16_t nvme_pid2ph(NvmeNamespace *ns, uint16_t pid)
345 {
346 uint16_t rgif = ns->endgrp->fdp.rgif;
347
348 if (!rgif) {
349 return pid;
350 }
351
352 return pid & ((1 << (15 - rgif)) - 1);
353 }
354
355 static inline uint16_t nvme_pid2rg(NvmeNamespace *ns, uint16_t pid)
356 {
357 uint16_t rgif = ns->endgrp->fdp.rgif;
358
359 if (!rgif) {
360 return 0;
361 }
362
363 return pid >> (16 - rgif);
364 }
365
366 static inline bool nvme_parse_pid(NvmeNamespace *ns, uint16_t pid,
367 uint16_t *ph, uint16_t *rg)
368 {
369 *rg = nvme_pid2rg(ns, pid);
370 *ph = nvme_pid2ph(ns, pid);
371
372 return nvme_ph_valid(ns, *ph) && nvme_rg_valid(ns->endgrp, *rg);
373 }
374
375 static void nvme_assign_zone_state(NvmeNamespace *ns, NvmeZone *zone,
376 NvmeZoneState state)
377 {
378 if (QTAILQ_IN_USE(zone, entry)) {
379 switch (nvme_get_zone_state(zone)) {
380 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
381 QTAILQ_REMOVE(&ns->exp_open_zones, zone, entry);
382 break;
383 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
384 QTAILQ_REMOVE(&ns->imp_open_zones, zone, entry);
385 break;
386 case NVME_ZONE_STATE_CLOSED:
387 QTAILQ_REMOVE(&ns->closed_zones, zone, entry);
388 break;
389 case NVME_ZONE_STATE_FULL:
390 QTAILQ_REMOVE(&ns->full_zones, zone, entry);
391 default:
392 ;
393 }
394 }
395
396 nvme_set_zone_state(zone, state);
397
398 switch (state) {
399 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
400 QTAILQ_INSERT_TAIL(&ns->exp_open_zones, zone, entry);
401 break;
402 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
403 QTAILQ_INSERT_TAIL(&ns->imp_open_zones, zone, entry);
404 break;
405 case NVME_ZONE_STATE_CLOSED:
406 QTAILQ_INSERT_TAIL(&ns->closed_zones, zone, entry);
407 break;
408 case NVME_ZONE_STATE_FULL:
409 QTAILQ_INSERT_TAIL(&ns->full_zones, zone, entry);
410 case NVME_ZONE_STATE_READ_ONLY:
411 break;
412 default:
413 zone->d.za = 0;
414 }
415 }
416
417 static uint16_t nvme_zns_check_resources(NvmeNamespace *ns, uint32_t act,
418 uint32_t opn, uint32_t zrwa)
419 {
420 if (ns->params.max_active_zones != 0 &&
421 ns->nr_active_zones + act > ns->params.max_active_zones) {
422 trace_pci_nvme_err_insuff_active_res(ns->params.max_active_zones);
423 return NVME_ZONE_TOO_MANY_ACTIVE | NVME_DNR;
424 }
425
426 if (ns->params.max_open_zones != 0 &&
427 ns->nr_open_zones + opn > ns->params.max_open_zones) {
428 trace_pci_nvme_err_insuff_open_res(ns->params.max_open_zones);
429 return NVME_ZONE_TOO_MANY_OPEN | NVME_DNR;
430 }
431
432 if (zrwa > ns->zns.numzrwa) {
433 return NVME_NOZRWA | NVME_DNR;
434 }
435
436 return NVME_SUCCESS;
437 }
438
439 /*
440 * Check if we can open a zone without exceeding open/active limits.
441 * AOR stands for "Active and Open Resources" (see TP 4053 section 2.5).
442 */
443 static uint16_t nvme_aor_check(NvmeNamespace *ns, uint32_t act, uint32_t opn)
444 {
445 return nvme_zns_check_resources(ns, act, opn, 0);
446 }
447
448 static NvmeFdpEvent *nvme_fdp_alloc_event(NvmeCtrl *n, NvmeFdpEventBuffer *ebuf)
449 {
450 NvmeFdpEvent *ret = NULL;
451 bool is_full = ebuf->next == ebuf->start && ebuf->nelems;
452
453 ret = &ebuf->events[ebuf->next++];
454 if (unlikely(ebuf->next == NVME_FDP_MAX_EVENTS)) {
455 ebuf->next = 0;
456 }
457 if (is_full) {
458 ebuf->start = ebuf->next;
459 } else {
460 ebuf->nelems++;
461 }
462
463 memset(ret, 0, sizeof(NvmeFdpEvent));
464 ret->timestamp = nvme_get_timestamp(n);
465
466 return ret;
467 }
468
469 static inline int log_event(NvmeRuHandle *ruh, uint8_t event_type)
470 {
471 return (ruh->event_filter >> nvme_fdp_evf_shifts[event_type]) & 0x1;
472 }
473
474 static bool nvme_update_ruh(NvmeCtrl *n, NvmeNamespace *ns, uint16_t pid)
475 {
476 NvmeEnduranceGroup *endgrp = ns->endgrp;
477 NvmeRuHandle *ruh;
478 NvmeReclaimUnit *ru;
479 NvmeFdpEvent *e = NULL;
480 uint16_t ph, rg, ruhid;
481
482 if (!nvme_parse_pid(ns, pid, &ph, &rg)) {
483 return false;
484 }
485
486 ruhid = ns->fdp.phs[ph];
487
488 ruh = &endgrp->fdp.ruhs[ruhid];
489 ru = &ruh->rus[rg];
490
491 if (ru->ruamw) {
492 if (log_event(ruh, FDP_EVT_RU_NOT_FULLY_WRITTEN)) {
493 e = nvme_fdp_alloc_event(n, &endgrp->fdp.host_events);
494 e->type = FDP_EVT_RU_NOT_FULLY_WRITTEN;
495 e->flags = FDPEF_PIV | FDPEF_NSIDV | FDPEF_LV;
496 e->pid = cpu_to_le16(pid);
497 e->nsid = cpu_to_le32(ns->params.nsid);
498 e->rgid = cpu_to_le16(rg);
499 e->ruhid = cpu_to_le16(ruhid);
500 }
501
502 /* log (eventual) GC overhead of prematurely swapping the RU */
503 nvme_fdp_stat_inc(&endgrp->fdp.mbmw, nvme_l2b(ns, ru->ruamw));
504 }
505
506 ru->ruamw = ruh->ruamw;
507
508 return true;
509 }
510
511 static bool nvme_addr_is_cmb(NvmeCtrl *n, hwaddr addr)
512 {
513 hwaddr hi, lo;
514
515 if (!n->cmb.cmse) {
516 return false;
517 }
518
519 lo = n->params.legacy_cmb ? n->cmb.mem.addr : n->cmb.cba;
520 hi = lo + int128_get64(n->cmb.mem.size);
521
522 return addr >= lo && addr < hi;
523 }
524
525 static inline void *nvme_addr_to_cmb(NvmeCtrl *n, hwaddr addr)
526 {
527 hwaddr base = n->params.legacy_cmb ? n->cmb.mem.addr : n->cmb.cba;
528 return &n->cmb.buf[addr - base];
529 }
530
531 static bool nvme_addr_is_pmr(NvmeCtrl *n, hwaddr addr)
532 {
533 hwaddr hi;
534
535 if (!n->pmr.cmse) {
536 return false;
537 }
538
539 hi = n->pmr.cba + int128_get64(n->pmr.dev->mr.size);
540
541 return addr >= n->pmr.cba && addr < hi;
542 }
543
544 static inline void *nvme_addr_to_pmr(NvmeCtrl *n, hwaddr addr)
545 {
546 return memory_region_get_ram_ptr(&n->pmr.dev->mr) + (addr - n->pmr.cba);
547 }
548
549 static inline bool nvme_addr_is_iomem(NvmeCtrl *n, hwaddr addr)
550 {
551 hwaddr hi, lo;
552
553 /*
554 * The purpose of this check is to guard against invalid "local" access to
555 * the iomem (i.e. controller registers). Thus, we check against the range
556 * covered by the 'bar0' MemoryRegion since that is currently composed of
557 * two subregions (the NVMe "MBAR" and the MSI-X table/pba). Note, however,
558 * that if the device model is ever changed to allow the CMB to be located
559 * in BAR0 as well, then this must be changed.
560 */
561 lo = n->bar0.addr;
562 hi = lo + int128_get64(n->bar0.size);
563
564 return addr >= lo && addr < hi;
565 }
566
567 static int nvme_addr_read(NvmeCtrl *n, hwaddr addr, void *buf, int size)
568 {
569 hwaddr hi = addr + size - 1;
570 if (hi < addr) {
571 return 1;
572 }
573
574 if (n->bar.cmbsz && nvme_addr_is_cmb(n, addr) && nvme_addr_is_cmb(n, hi)) {
575 memcpy(buf, nvme_addr_to_cmb(n, addr), size);
576 return 0;
577 }
578
579 if (nvme_addr_is_pmr(n, addr) && nvme_addr_is_pmr(n, hi)) {
580 memcpy(buf, nvme_addr_to_pmr(n, addr), size);
581 return 0;
582 }
583
584 return pci_dma_read(PCI_DEVICE(n), addr, buf, size);
585 }
586
587 static int nvme_addr_write(NvmeCtrl *n, hwaddr addr, const void *buf, int size)
588 {
589 hwaddr hi = addr + size - 1;
590 if (hi < addr) {
591 return 1;
592 }
593
594 if (n->bar.cmbsz && nvme_addr_is_cmb(n, addr) && nvme_addr_is_cmb(n, hi)) {
595 memcpy(nvme_addr_to_cmb(n, addr), buf, size);
596 return 0;
597 }
598
599 if (nvme_addr_is_pmr(n, addr) && nvme_addr_is_pmr(n, hi)) {
600 memcpy(nvme_addr_to_pmr(n, addr), buf, size);
601 return 0;
602 }
603
604 return pci_dma_write(PCI_DEVICE(n), addr, buf, size);
605 }
606
607 static bool nvme_nsid_valid(NvmeCtrl *n, uint32_t nsid)
608 {
609 return nsid &&
610 (nsid == NVME_NSID_BROADCAST || nsid <= NVME_MAX_NAMESPACES);
611 }
612
613 static int nvme_check_sqid(NvmeCtrl *n, uint16_t sqid)
614 {
615 return sqid < n->conf_ioqpairs + 1 && n->sq[sqid] != NULL ? 0 : -1;
616 }
617
618 static int nvme_check_cqid(NvmeCtrl *n, uint16_t cqid)
619 {
620 return cqid < n->conf_ioqpairs + 1 && n->cq[cqid] != NULL ? 0 : -1;
621 }
622
623 static void nvme_inc_cq_tail(NvmeCQueue *cq)
624 {
625 cq->tail++;
626 if (cq->tail >= cq->size) {
627 cq->tail = 0;
628 cq->phase = !cq->phase;
629 }
630 }
631
632 static void nvme_inc_sq_head(NvmeSQueue *sq)
633 {
634 sq->head = (sq->head + 1) % sq->size;
635 }
636
637 static uint8_t nvme_cq_full(NvmeCQueue *cq)
638 {
639 return (cq->tail + 1) % cq->size == cq->head;
640 }
641
642 static uint8_t nvme_sq_empty(NvmeSQueue *sq)
643 {
644 return sq->head == sq->tail;
645 }
646
647 static void nvme_irq_check(NvmeCtrl *n)
648 {
649 PCIDevice *pci = PCI_DEVICE(n);
650 uint32_t intms = ldl_le_p(&n->bar.intms);
651
652 if (msix_enabled(pci)) {
653 return;
654 }
655 if (~intms & n->irq_status) {
656 pci_irq_assert(pci);
657 } else {
658 pci_irq_deassert(pci);
659 }
660 }
661
662 static void nvme_irq_assert(NvmeCtrl *n, NvmeCQueue *cq)
663 {
664 PCIDevice *pci = PCI_DEVICE(n);
665
666 if (cq->irq_enabled) {
667 if (msix_enabled(pci)) {
668 trace_pci_nvme_irq_msix(cq->vector);
669 msix_notify(pci, cq->vector);
670 } else {
671 trace_pci_nvme_irq_pin();
672 assert(cq->vector < 32);
673 n->irq_status |= 1 << cq->vector;
674 nvme_irq_check(n);
675 }
676 } else {
677 trace_pci_nvme_irq_masked();
678 }
679 }
680
681 static void nvme_irq_deassert(NvmeCtrl *n, NvmeCQueue *cq)
682 {
683 if (cq->irq_enabled) {
684 if (msix_enabled(PCI_DEVICE(n))) {
685 return;
686 } else {
687 assert(cq->vector < 32);
688 if (!n->cq_pending) {
689 n->irq_status &= ~(1 << cq->vector);
690 }
691 nvme_irq_check(n);
692 }
693 }
694 }
695
696 static void nvme_req_clear(NvmeRequest *req)
697 {
698 req->ns = NULL;
699 req->opaque = NULL;
700 req->aiocb = NULL;
701 memset(&req->cqe, 0x0, sizeof(req->cqe));
702 req->status = NVME_SUCCESS;
703 }
704
705 static inline void nvme_sg_init(NvmeCtrl *n, NvmeSg *sg, bool dma)
706 {
707 if (dma) {
708 pci_dma_sglist_init(&sg->qsg, PCI_DEVICE(n), 0);
709 sg->flags = NVME_SG_DMA;
710 } else {
711 qemu_iovec_init(&sg->iov, 0);
712 }
713
714 sg->flags |= NVME_SG_ALLOC;
715 }
716
717 static inline void nvme_sg_unmap(NvmeSg *sg)
718 {
719 if (!(sg->flags & NVME_SG_ALLOC)) {
720 return;
721 }
722
723 if (sg->flags & NVME_SG_DMA) {
724 qemu_sglist_destroy(&sg->qsg);
725 } else {
726 qemu_iovec_destroy(&sg->iov);
727 }
728
729 memset(sg, 0x0, sizeof(*sg));
730 }
731
732 /*
733 * When metadata is transfered as extended LBAs, the DPTR mapped into `sg`
734 * holds both data and metadata. This function splits the data and metadata
735 * into two separate QSG/IOVs.
736 */
737 static void nvme_sg_split(NvmeSg *sg, NvmeNamespace *ns, NvmeSg *data,
738 NvmeSg *mdata)
739 {
740 NvmeSg *dst = data;
741 uint32_t trans_len, count = ns->lbasz;
742 uint64_t offset = 0;
743 bool dma = sg->flags & NVME_SG_DMA;
744 size_t sge_len;
745 size_t sg_len = dma ? sg->qsg.size : sg->iov.size;
746 int sg_idx = 0;
747
748 assert(sg->flags & NVME_SG_ALLOC);
749
750 while (sg_len) {
751 sge_len = dma ? sg->qsg.sg[sg_idx].len : sg->iov.iov[sg_idx].iov_len;
752
753 trans_len = MIN(sg_len, count);
754 trans_len = MIN(trans_len, sge_len - offset);
755
756 if (dst) {
757 if (dma) {
758 qemu_sglist_add(&dst->qsg, sg->qsg.sg[sg_idx].base + offset,
759 trans_len);
760 } else {
761 qemu_iovec_add(&dst->iov,
762 sg->iov.iov[sg_idx].iov_base + offset,
763 trans_len);
764 }
765 }
766
767 sg_len -= trans_len;
768 count -= trans_len;
769 offset += trans_len;
770
771 if (count == 0) {
772 dst = (dst == data) ? mdata : data;
773 count = (dst == data) ? ns->lbasz : ns->lbaf.ms;
774 }
775
776 if (sge_len == offset) {
777 offset = 0;
778 sg_idx++;
779 }
780 }
781 }
782
783 static uint16_t nvme_map_addr_cmb(NvmeCtrl *n, QEMUIOVector *iov, hwaddr addr,
784 size_t len)
785 {
786 if (!len) {
787 return NVME_SUCCESS;
788 }
789
790 trace_pci_nvme_map_addr_cmb(addr, len);
791
792 if (!nvme_addr_is_cmb(n, addr) || !nvme_addr_is_cmb(n, addr + len - 1)) {
793 return NVME_DATA_TRAS_ERROR;
794 }
795
796 qemu_iovec_add(iov, nvme_addr_to_cmb(n, addr), len);
797
798 return NVME_SUCCESS;
799 }
800
801 static uint16_t nvme_map_addr_pmr(NvmeCtrl *n, QEMUIOVector *iov, hwaddr addr,
802 size_t len)
803 {
804 if (!len) {
805 return NVME_SUCCESS;
806 }
807
808 if (!nvme_addr_is_pmr(n, addr) || !nvme_addr_is_pmr(n, addr + len - 1)) {
809 return NVME_DATA_TRAS_ERROR;
810 }
811
812 qemu_iovec_add(iov, nvme_addr_to_pmr(n, addr), len);
813
814 return NVME_SUCCESS;
815 }
816
817 static uint16_t nvme_map_addr(NvmeCtrl *n, NvmeSg *sg, hwaddr addr, size_t len)
818 {
819 bool cmb = false, pmr = false;
820
821 if (!len) {
822 return NVME_SUCCESS;
823 }
824
825 trace_pci_nvme_map_addr(addr, len);
826
827 if (nvme_addr_is_iomem(n, addr)) {
828 return NVME_DATA_TRAS_ERROR;
829 }
830
831 if (nvme_addr_is_cmb(n, addr)) {
832 cmb = true;
833 } else if (nvme_addr_is_pmr(n, addr)) {
834 pmr = true;
835 }
836
837 if (cmb || pmr) {
838 if (sg->flags & NVME_SG_DMA) {
839 return NVME_INVALID_USE_OF_CMB | NVME_DNR;
840 }
841
842 if (sg->iov.niov + 1 > IOV_MAX) {
843 goto max_mappings_exceeded;
844 }
845
846 if (cmb) {
847 return nvme_map_addr_cmb(n, &sg->iov, addr, len);
848 } else {
849 return nvme_map_addr_pmr(n, &sg->iov, addr, len);
850 }
851 }
852
853 if (!(sg->flags & NVME_SG_DMA)) {
854 return NVME_INVALID_USE_OF_CMB | NVME_DNR;
855 }
856
857 if (sg->qsg.nsg + 1 > IOV_MAX) {
858 goto max_mappings_exceeded;
859 }
860
861 qemu_sglist_add(&sg->qsg, addr, len);
862
863 return NVME_SUCCESS;
864
865 max_mappings_exceeded:
866 NVME_GUEST_ERR(pci_nvme_ub_too_many_mappings,
867 "number of mappings exceed 1024");
868 return NVME_INTERNAL_DEV_ERROR | NVME_DNR;
869 }
870
871 static inline bool nvme_addr_is_dma(NvmeCtrl *n, hwaddr addr)
872 {
873 return !(nvme_addr_is_cmb(n, addr) || nvme_addr_is_pmr(n, addr));
874 }
875
876 static uint16_t nvme_map_prp(NvmeCtrl *n, NvmeSg *sg, uint64_t prp1,
877 uint64_t prp2, uint32_t len)
878 {
879 hwaddr trans_len = n->page_size - (prp1 % n->page_size);
880 trans_len = MIN(len, trans_len);
881 int num_prps = (len >> n->page_bits) + 1;
882 uint16_t status;
883 int ret;
884
885 trace_pci_nvme_map_prp(trans_len, len, prp1, prp2, num_prps);
886
887 nvme_sg_init(n, sg, nvme_addr_is_dma(n, prp1));
888
889 status = nvme_map_addr(n, sg, prp1, trans_len);
890 if (status) {
891 goto unmap;
892 }
893
894 len -= trans_len;
895 if (len) {
896 if (len > n->page_size) {
897 uint64_t prp_list[n->max_prp_ents];
898 uint32_t nents, prp_trans;
899 int i = 0;
900
901 /*
902 * The first PRP list entry, pointed to by PRP2 may contain offset.
903 * Hence, we need to calculate the number of entries in based on
904 * that offset.
905 */
906 nents = (n->page_size - (prp2 & (n->page_size - 1))) >> 3;
907 prp_trans = MIN(n->max_prp_ents, nents) * sizeof(uint64_t);
908 ret = nvme_addr_read(n, prp2, (void *)prp_list, prp_trans);
909 if (ret) {
910 trace_pci_nvme_err_addr_read(prp2);
911 status = NVME_DATA_TRAS_ERROR;
912 goto unmap;
913 }
914 while (len != 0) {
915 uint64_t prp_ent = le64_to_cpu(prp_list[i]);
916
917 if (i == nents - 1 && len > n->page_size) {
918 if (unlikely(prp_ent & (n->page_size - 1))) {
919 trace_pci_nvme_err_invalid_prplist_ent(prp_ent);
920 status = NVME_INVALID_PRP_OFFSET | NVME_DNR;
921 goto unmap;
922 }
923
924 i = 0;
925 nents = (len + n->page_size - 1) >> n->page_bits;
926 nents = MIN(nents, n->max_prp_ents);
927 prp_trans = nents * sizeof(uint64_t);
928 ret = nvme_addr_read(n, prp_ent, (void *)prp_list,
929 prp_trans);
930 if (ret) {
931 trace_pci_nvme_err_addr_read(prp_ent);
932 status = NVME_DATA_TRAS_ERROR;
933 goto unmap;
934 }
935 prp_ent = le64_to_cpu(prp_list[i]);
936 }
937
938 if (unlikely(prp_ent & (n->page_size - 1))) {
939 trace_pci_nvme_err_invalid_prplist_ent(prp_ent);
940 status = NVME_INVALID_PRP_OFFSET | NVME_DNR;
941 goto unmap;
942 }
943
944 trans_len = MIN(len, n->page_size);
945 status = nvme_map_addr(n, sg, prp_ent, trans_len);
946 if (status) {
947 goto unmap;
948 }
949
950 len -= trans_len;
951 i++;
952 }
953 } else {
954 if (unlikely(prp2 & (n->page_size - 1))) {
955 trace_pci_nvme_err_invalid_prp2_align(prp2);
956 status = NVME_INVALID_PRP_OFFSET | NVME_DNR;
957 goto unmap;
958 }
959 status = nvme_map_addr(n, sg, prp2, len);
960 if (status) {
961 goto unmap;
962 }
963 }
964 }
965
966 return NVME_SUCCESS;
967
968 unmap:
969 nvme_sg_unmap(sg);
970 return status;
971 }
972
973 /*
974 * Map 'nsgld' data descriptors from 'segment'. The function will subtract the
975 * number of bytes mapped in len.
976 */
977 static uint16_t nvme_map_sgl_data(NvmeCtrl *n, NvmeSg *sg,
978 NvmeSglDescriptor *segment, uint64_t nsgld,
979 size_t *len, NvmeCmd *cmd)
980 {
981 dma_addr_t addr, trans_len;
982 uint32_t dlen;
983 uint16_t status;
984
985 for (int i = 0; i < nsgld; i++) {
986 uint8_t type = NVME_SGL_TYPE(segment[i].type);
987
988 switch (type) {
989 case NVME_SGL_DESCR_TYPE_DATA_BLOCK:
990 break;
991 case NVME_SGL_DESCR_TYPE_SEGMENT:
992 case NVME_SGL_DESCR_TYPE_LAST_SEGMENT:
993 return NVME_INVALID_NUM_SGL_DESCRS | NVME_DNR;
994 default:
995 return NVME_SGL_DESCR_TYPE_INVALID | NVME_DNR;
996 }
997
998 dlen = le32_to_cpu(segment[i].len);
999
1000 if (!dlen) {
1001 continue;
1002 }
1003
1004 if (*len == 0) {
1005 /*
1006 * All data has been mapped, but the SGL contains additional
1007 * segments and/or descriptors. The controller might accept
1008 * ignoring the rest of the SGL.
1009 */
1010 uint32_t sgls = le32_to_cpu(n->id_ctrl.sgls);
1011 if (sgls & NVME_CTRL_SGLS_EXCESS_LENGTH) {
1012 break;
1013 }
1014
1015 trace_pci_nvme_err_invalid_sgl_excess_length(dlen);
1016 return NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
1017 }
1018
1019 trans_len = MIN(*len, dlen);
1020
1021 addr = le64_to_cpu(segment[i].addr);
1022
1023 if (UINT64_MAX - addr < dlen) {
1024 return NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
1025 }
1026
1027 status = nvme_map_addr(n, sg, addr, trans_len);
1028 if (status) {
1029 return status;
1030 }
1031
1032 *len -= trans_len;
1033 }
1034
1035 return NVME_SUCCESS;
1036 }
1037
1038 static uint16_t nvme_map_sgl(NvmeCtrl *n, NvmeSg *sg, NvmeSglDescriptor sgl,
1039 size_t len, NvmeCmd *cmd)
1040 {
1041 /*
1042 * Read the segment in chunks of 256 descriptors (one 4k page) to avoid
1043 * dynamically allocating a potentially huge SGL. The spec allows the SGL
1044 * to be larger (as in number of bytes required to describe the SGL
1045 * descriptors and segment chain) than the command transfer size, so it is
1046 * not bounded by MDTS.
1047 */
1048 const int SEG_CHUNK_SIZE = 256;
1049
1050 NvmeSglDescriptor segment[SEG_CHUNK_SIZE], *sgld, *last_sgld;
1051 uint64_t nsgld;
1052 uint32_t seg_len;
1053 uint16_t status;
1054 hwaddr addr;
1055 int ret;
1056
1057 sgld = &sgl;
1058 addr = le64_to_cpu(sgl.addr);
1059
1060 trace_pci_nvme_map_sgl(NVME_SGL_TYPE(sgl.type), len);
1061
1062 nvme_sg_init(n, sg, nvme_addr_is_dma(n, addr));
1063
1064 /*
1065 * If the entire transfer can be described with a single data block it can
1066 * be mapped directly.
1067 */
1068 if (NVME_SGL_TYPE(sgl.type) == NVME_SGL_DESCR_TYPE_DATA_BLOCK) {
1069 status = nvme_map_sgl_data(n, sg, sgld, 1, &len, cmd);
1070 if (status) {
1071 goto unmap;
1072 }
1073
1074 goto out;
1075 }
1076
1077 for (;;) {
1078 switch (NVME_SGL_TYPE(sgld->type)) {
1079 case NVME_SGL_DESCR_TYPE_SEGMENT:
1080 case NVME_SGL_DESCR_TYPE_LAST_SEGMENT:
1081 break;
1082 default:
1083 return NVME_INVALID_SGL_SEG_DESCR | NVME_DNR;
1084 }
1085
1086 seg_len = le32_to_cpu(sgld->len);
1087
1088 /* check the length of the (Last) Segment descriptor */
1089 if (!seg_len || seg_len & 0xf) {
1090 return NVME_INVALID_SGL_SEG_DESCR | NVME_DNR;
1091 }
1092
1093 if (UINT64_MAX - addr < seg_len) {
1094 return NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
1095 }
1096
1097 nsgld = seg_len / sizeof(NvmeSglDescriptor);
1098
1099 while (nsgld > SEG_CHUNK_SIZE) {
1100 if (nvme_addr_read(n, addr, segment, sizeof(segment))) {
1101 trace_pci_nvme_err_addr_read(addr);
1102 status = NVME_DATA_TRAS_ERROR;
1103 goto unmap;
1104 }
1105
1106 status = nvme_map_sgl_data(n, sg, segment, SEG_CHUNK_SIZE,
1107 &len, cmd);
1108 if (status) {
1109 goto unmap;
1110 }
1111
1112 nsgld -= SEG_CHUNK_SIZE;
1113 addr += SEG_CHUNK_SIZE * sizeof(NvmeSglDescriptor);
1114 }
1115
1116 ret = nvme_addr_read(n, addr, segment, nsgld *
1117 sizeof(NvmeSglDescriptor));
1118 if (ret) {
1119 trace_pci_nvme_err_addr_read(addr);
1120 status = NVME_DATA_TRAS_ERROR;
1121 goto unmap;
1122 }
1123
1124 last_sgld = &segment[nsgld - 1];
1125
1126 /*
1127 * If the segment ends with a Data Block, then we are done.
1128 */
1129 if (NVME_SGL_TYPE(last_sgld->type) == NVME_SGL_DESCR_TYPE_DATA_BLOCK) {
1130 status = nvme_map_sgl_data(n, sg, segment, nsgld, &len, cmd);
1131 if (status) {
1132 goto unmap;
1133 }
1134
1135 goto out;
1136 }
1137
1138 /*
1139 * If the last descriptor was not a Data Block, then the current
1140 * segment must not be a Last Segment.
1141 */
1142 if (NVME_SGL_TYPE(sgld->type) == NVME_SGL_DESCR_TYPE_LAST_SEGMENT) {
1143 status = NVME_INVALID_SGL_SEG_DESCR | NVME_DNR;
1144 goto unmap;
1145 }
1146
1147 sgld = last_sgld;
1148 addr = le64_to_cpu(sgld->addr);
1149
1150 /*
1151 * Do not map the last descriptor; it will be a Segment or Last Segment
1152 * descriptor and is handled by the next iteration.
1153 */
1154 status = nvme_map_sgl_data(n, sg, segment, nsgld - 1, &len, cmd);
1155 if (status) {
1156 goto unmap;
1157 }
1158 }
1159
1160 out:
1161 /* if there is any residual left in len, the SGL was too short */
1162 if (len) {
1163 status = NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
1164 goto unmap;
1165 }
1166
1167 return NVME_SUCCESS;
1168
1169 unmap:
1170 nvme_sg_unmap(sg);
1171 return status;
1172 }
1173
1174 uint16_t nvme_map_dptr(NvmeCtrl *n, NvmeSg *sg, size_t len,
1175 NvmeCmd *cmd)
1176 {
1177 uint64_t prp1, prp2;
1178
1179 switch (NVME_CMD_FLAGS_PSDT(cmd->flags)) {
1180 case NVME_PSDT_PRP:
1181 prp1 = le64_to_cpu(cmd->dptr.prp1);
1182 prp2 = le64_to_cpu(cmd->dptr.prp2);
1183
1184 return nvme_map_prp(n, sg, prp1, prp2, len);
1185 case NVME_PSDT_SGL_MPTR_CONTIGUOUS:
1186 case NVME_PSDT_SGL_MPTR_SGL:
1187 return nvme_map_sgl(n, sg, cmd->dptr.sgl, len, cmd);
1188 default:
1189 return NVME_INVALID_FIELD;
1190 }
1191 }
1192
1193 static uint16_t nvme_map_mptr(NvmeCtrl *n, NvmeSg *sg, size_t len,
1194 NvmeCmd *cmd)
1195 {
1196 int psdt = NVME_CMD_FLAGS_PSDT(cmd->flags);
1197 hwaddr mptr = le64_to_cpu(cmd->mptr);
1198 uint16_t status;
1199
1200 if (psdt == NVME_PSDT_SGL_MPTR_SGL) {
1201 NvmeSglDescriptor sgl;
1202
1203 if (nvme_addr_read(n, mptr, &sgl, sizeof(sgl))) {
1204 return NVME_DATA_TRAS_ERROR;
1205 }
1206
1207 status = nvme_map_sgl(n, sg, sgl, len, cmd);
1208 if (status && (status & 0x7ff) == NVME_DATA_SGL_LEN_INVALID) {
1209 status = NVME_MD_SGL_LEN_INVALID | NVME_DNR;
1210 }
1211
1212 return status;
1213 }
1214
1215 nvme_sg_init(n, sg, nvme_addr_is_dma(n, mptr));
1216 status = nvme_map_addr(n, sg, mptr, len);
1217 if (status) {
1218 nvme_sg_unmap(sg);
1219 }
1220
1221 return status;
1222 }
1223
1224 static uint16_t nvme_map_data(NvmeCtrl *n, uint32_t nlb, NvmeRequest *req)
1225 {
1226 NvmeNamespace *ns = req->ns;
1227 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
1228 bool pi = !!NVME_ID_NS_DPS_TYPE(ns->id_ns.dps);
1229 bool pract = !!(le16_to_cpu(rw->control) & NVME_RW_PRINFO_PRACT);
1230 size_t len = nvme_l2b(ns, nlb);
1231 uint16_t status;
1232
1233 if (nvme_ns_ext(ns) &&
1234 !(pi && pract && ns->lbaf.ms == nvme_pi_tuple_size(ns))) {
1235 NvmeSg sg;
1236
1237 len += nvme_m2b(ns, nlb);
1238
1239 status = nvme_map_dptr(n, &sg, len, &req->cmd);
1240 if (status) {
1241 return status;
1242 }
1243
1244 nvme_sg_init(n, &req->sg, sg.flags & NVME_SG_DMA);
1245 nvme_sg_split(&sg, ns, &req->sg, NULL);
1246 nvme_sg_unmap(&sg);
1247
1248 return NVME_SUCCESS;
1249 }
1250
1251 return nvme_map_dptr(n, &req->sg, len, &req->cmd);
1252 }
1253
1254 static uint16_t nvme_map_mdata(NvmeCtrl *n, uint32_t nlb, NvmeRequest *req)
1255 {
1256 NvmeNamespace *ns = req->ns;
1257 size_t len = nvme_m2b(ns, nlb);
1258 uint16_t status;
1259
1260 if (nvme_ns_ext(ns)) {
1261 NvmeSg sg;
1262
1263 len += nvme_l2b(ns, nlb);
1264
1265 status = nvme_map_dptr(n, &sg, len, &req->cmd);
1266 if (status) {
1267 return status;
1268 }
1269
1270 nvme_sg_init(n, &req->sg, sg.flags & NVME_SG_DMA);
1271 nvme_sg_split(&sg, ns, NULL, &req->sg);
1272 nvme_sg_unmap(&sg);
1273
1274 return NVME_SUCCESS;
1275 }
1276
1277 return nvme_map_mptr(n, &req->sg, len, &req->cmd);
1278 }
1279
1280 static uint16_t nvme_tx_interleaved(NvmeCtrl *n, NvmeSg *sg, uint8_t *ptr,
1281 uint32_t len, uint32_t bytes,
1282 int32_t skip_bytes, int64_t offset,
1283 NvmeTxDirection dir)
1284 {
1285 hwaddr addr;
1286 uint32_t trans_len, count = bytes;
1287 bool dma = sg->flags & NVME_SG_DMA;
1288 int64_t sge_len;
1289 int sg_idx = 0;
1290 int ret;
1291
1292 assert(sg->flags & NVME_SG_ALLOC);
1293
1294 while (len) {
1295 sge_len = dma ? sg->qsg.sg[sg_idx].len : sg->iov.iov[sg_idx].iov_len;
1296
1297 if (sge_len - offset < 0) {
1298 offset -= sge_len;
1299 sg_idx++;
1300 continue;
1301 }
1302
1303 if (sge_len == offset) {
1304 offset = 0;
1305 sg_idx++;
1306 continue;
1307 }
1308
1309 trans_len = MIN(len, count);
1310 trans_len = MIN(trans_len, sge_len - offset);
1311
1312 if (dma) {
1313 addr = sg->qsg.sg[sg_idx].base + offset;
1314 } else {
1315 addr = (hwaddr)(uintptr_t)sg->iov.iov[sg_idx].iov_base + offset;
1316 }
1317
1318 if (dir == NVME_TX_DIRECTION_TO_DEVICE) {
1319 ret = nvme_addr_read(n, addr, ptr, trans_len);
1320 } else {
1321 ret = nvme_addr_write(n, addr, ptr, trans_len);
1322 }
1323
1324 if (ret) {
1325 return NVME_DATA_TRAS_ERROR;
1326 }
1327
1328 ptr += trans_len;
1329 len -= trans_len;
1330 count -= trans_len;
1331 offset += trans_len;
1332
1333 if (count == 0) {
1334 count = bytes;
1335 offset += skip_bytes;
1336 }
1337 }
1338
1339 return NVME_SUCCESS;
1340 }
1341
1342 static uint16_t nvme_tx(NvmeCtrl *n, NvmeSg *sg, void *ptr, uint32_t len,
1343 NvmeTxDirection dir)
1344 {
1345 assert(sg->flags & NVME_SG_ALLOC);
1346
1347 if (sg->flags & NVME_SG_DMA) {
1348 const MemTxAttrs attrs = MEMTXATTRS_UNSPECIFIED;
1349 dma_addr_t residual;
1350
1351 if (dir == NVME_TX_DIRECTION_TO_DEVICE) {
1352 dma_buf_write(ptr, len, &residual, &sg->qsg, attrs);
1353 } else {
1354 dma_buf_read(ptr, len, &residual, &sg->qsg, attrs);
1355 }
1356
1357 if (unlikely(residual)) {
1358 trace_pci_nvme_err_invalid_dma();
1359 return NVME_INVALID_FIELD | NVME_DNR;
1360 }
1361 } else {
1362 size_t bytes;
1363
1364 if (dir == NVME_TX_DIRECTION_TO_DEVICE) {
1365 bytes = qemu_iovec_to_buf(&sg->iov, 0, ptr, len);
1366 } else {
1367 bytes = qemu_iovec_from_buf(&sg->iov, 0, ptr, len);
1368 }
1369
1370 if (unlikely(bytes != len)) {
1371 trace_pci_nvme_err_invalid_dma();
1372 return NVME_INVALID_FIELD | NVME_DNR;
1373 }
1374 }
1375
1376 return NVME_SUCCESS;
1377 }
1378
1379 static inline uint16_t nvme_c2h(NvmeCtrl *n, void *ptr, uint32_t len,
1380 NvmeRequest *req)
1381 {
1382 uint16_t status;
1383
1384 status = nvme_map_dptr(n, &req->sg, len, &req->cmd);
1385 if (status) {
1386 return status;
1387 }
1388
1389 return nvme_tx(n, &req->sg, ptr, len, NVME_TX_DIRECTION_FROM_DEVICE);
1390 }
1391
1392 static inline uint16_t nvme_h2c(NvmeCtrl *n, void *ptr, uint32_t len,
1393 NvmeRequest *req)
1394 {
1395 uint16_t status;
1396
1397 status = nvme_map_dptr(n, &req->sg, len, &req->cmd);
1398 if (status) {
1399 return status;
1400 }
1401
1402 return nvme_tx(n, &req->sg, ptr, len, NVME_TX_DIRECTION_TO_DEVICE);
1403 }
1404
1405 uint16_t nvme_bounce_data(NvmeCtrl *n, void *ptr, uint32_t len,
1406 NvmeTxDirection dir, NvmeRequest *req)
1407 {
1408 NvmeNamespace *ns = req->ns;
1409 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
1410 bool pi = !!NVME_ID_NS_DPS_TYPE(ns->id_ns.dps);
1411 bool pract = !!(le16_to_cpu(rw->control) & NVME_RW_PRINFO_PRACT);
1412
1413 if (nvme_ns_ext(ns) &&
1414 !(pi && pract && ns->lbaf.ms == nvme_pi_tuple_size(ns))) {
1415 return nvme_tx_interleaved(n, &req->sg, ptr, len, ns->lbasz,
1416 ns->lbaf.ms, 0, dir);
1417 }
1418
1419 return nvme_tx(n, &req->sg, ptr, len, dir);
1420 }
1421
1422 uint16_t nvme_bounce_mdata(NvmeCtrl *n, void *ptr, uint32_t len,
1423 NvmeTxDirection dir, NvmeRequest *req)
1424 {
1425 NvmeNamespace *ns = req->ns;
1426 uint16_t status;
1427
1428 if (nvme_ns_ext(ns)) {
1429 return nvme_tx_interleaved(n, &req->sg, ptr, len, ns->lbaf.ms,
1430 ns->lbasz, ns->lbasz, dir);
1431 }
1432
1433 nvme_sg_unmap(&req->sg);
1434
1435 status = nvme_map_mptr(n, &req->sg, len, &req->cmd);
1436 if (status) {
1437 return status;
1438 }
1439
1440 return nvme_tx(n, &req->sg, ptr, len, dir);
1441 }
1442
1443 static inline void nvme_blk_read(BlockBackend *blk, int64_t offset,
1444 uint32_t align, BlockCompletionFunc *cb,
1445 NvmeRequest *req)
1446 {
1447 assert(req->sg.flags & NVME_SG_ALLOC);
1448
1449 if (req->sg.flags & NVME_SG_DMA) {
1450 req->aiocb = dma_blk_read(blk, &req->sg.qsg, offset, align, cb, req);
1451 } else {
1452 req->aiocb = blk_aio_preadv(blk, offset, &req->sg.iov, 0, cb, req);
1453 }
1454 }
1455
1456 static inline void nvme_blk_write(BlockBackend *blk, int64_t offset,
1457 uint32_t align, BlockCompletionFunc *cb,
1458 NvmeRequest *req)
1459 {
1460 assert(req->sg.flags & NVME_SG_ALLOC);
1461
1462 if (req->sg.flags & NVME_SG_DMA) {
1463 req->aiocb = dma_blk_write(blk, &req->sg.qsg, offset, align, cb, req);
1464 } else {
1465 req->aiocb = blk_aio_pwritev(blk, offset, &req->sg.iov, 0, cb, req);
1466 }
1467 }
1468
1469 static void nvme_update_cq_eventidx(const NvmeCQueue *cq)
1470 {
1471 uint32_t v = cpu_to_le32(cq->head);
1472
1473 trace_pci_nvme_update_cq_eventidx(cq->cqid, cq->head);
1474
1475 pci_dma_write(PCI_DEVICE(cq->ctrl), cq->ei_addr, &v, sizeof(v));
1476 }
1477
1478 static void nvme_update_cq_head(NvmeCQueue *cq)
1479 {
1480 uint32_t v;
1481
1482 pci_dma_read(PCI_DEVICE(cq->ctrl), cq->db_addr, &v, sizeof(v));
1483
1484 cq->head = le32_to_cpu(v);
1485
1486 trace_pci_nvme_update_cq_head(cq->cqid, cq->head);
1487 }
1488
1489 static void nvme_post_cqes(void *opaque)
1490 {
1491 NvmeCQueue *cq = opaque;
1492 NvmeCtrl *n = cq->ctrl;
1493 NvmeRequest *req, *next;
1494 bool pending = cq->head != cq->tail;
1495 int ret;
1496
1497 QTAILQ_FOREACH_SAFE(req, &cq->req_list, entry, next) {
1498 NvmeSQueue *sq;
1499 hwaddr addr;
1500
1501 if (n->dbbuf_enabled) {
1502 nvme_update_cq_eventidx(cq);
1503 nvme_update_cq_head(cq);
1504 }
1505
1506 if (nvme_cq_full(cq)) {
1507 break;
1508 }
1509
1510 sq = req->sq;
1511 req->cqe.status = cpu_to_le16((req->status << 1) | cq->phase);
1512 req->cqe.sq_id = cpu_to_le16(sq->sqid);
1513 req->cqe.sq_head = cpu_to_le16(sq->head);
1514 addr = cq->dma_addr + cq->tail * n->cqe_size;
1515 ret = pci_dma_write(PCI_DEVICE(n), addr, (void *)&req->cqe,
1516 sizeof(req->cqe));
1517 if (ret) {
1518 trace_pci_nvme_err_addr_write(addr);
1519 trace_pci_nvme_err_cfs();
1520 stl_le_p(&n->bar.csts, NVME_CSTS_FAILED);
1521 break;
1522 }
1523 QTAILQ_REMOVE(&cq->req_list, req, entry);
1524 nvme_inc_cq_tail(cq);
1525 nvme_sg_unmap(&req->sg);
1526 QTAILQ_INSERT_TAIL(&sq->req_list, req, entry);
1527 }
1528 if (cq->tail != cq->head) {
1529 if (cq->irq_enabled && !pending) {
1530 n->cq_pending++;
1531 }
1532
1533 nvme_irq_assert(n, cq);
1534 }
1535 }
1536
1537 static void nvme_enqueue_req_completion(NvmeCQueue *cq, NvmeRequest *req)
1538 {
1539 assert(cq->cqid == req->sq->cqid);
1540 trace_pci_nvme_enqueue_req_completion(nvme_cid(req), cq->cqid,
1541 le32_to_cpu(req->cqe.result),
1542 le32_to_cpu(req->cqe.dw1),
1543 req->status);
1544
1545 if (req->status) {
1546 trace_pci_nvme_err_req_status(nvme_cid(req), nvme_nsid(req->ns),
1547 req->status, req->cmd.opcode);
1548 }
1549
1550 QTAILQ_REMOVE(&req->sq->out_req_list, req, entry);
1551 QTAILQ_INSERT_TAIL(&cq->req_list, req, entry);
1552
1553 qemu_bh_schedule(cq->bh);
1554 }
1555
1556 static void nvme_process_aers(void *opaque)
1557 {
1558 NvmeCtrl *n = opaque;
1559 NvmeAsyncEvent *event, *next;
1560
1561 trace_pci_nvme_process_aers(n->aer_queued);
1562
1563 QTAILQ_FOREACH_SAFE(event, &n->aer_queue, entry, next) {
1564 NvmeRequest *req;
1565 NvmeAerResult *result;
1566
1567 /* can't post cqe if there is nothing to complete */
1568 if (!n->outstanding_aers) {
1569 trace_pci_nvme_no_outstanding_aers();
1570 break;
1571 }
1572
1573 /* ignore if masked (cqe posted, but event not cleared) */
1574 if (n->aer_mask & (1 << event->result.event_type)) {
1575 trace_pci_nvme_aer_masked(event->result.event_type, n->aer_mask);
1576 continue;
1577 }
1578
1579 QTAILQ_REMOVE(&n->aer_queue, event, entry);
1580 n->aer_queued--;
1581
1582 n->aer_mask |= 1 << event->result.event_type;
1583 n->outstanding_aers--;
1584
1585 req = n->aer_reqs[n->outstanding_aers];
1586
1587 result = (NvmeAerResult *) &req->cqe.result;
1588 result->event_type = event->result.event_type;
1589 result->event_info = event->result.event_info;
1590 result->log_page = event->result.log_page;
1591 g_free(event);
1592
1593 trace_pci_nvme_aer_post_cqe(result->event_type, result->event_info,
1594 result->log_page);
1595
1596 nvme_enqueue_req_completion(&n->admin_cq, req);
1597 }
1598 }
1599
1600 static void nvme_enqueue_event(NvmeCtrl *n, uint8_t event_type,
1601 uint8_t event_info, uint8_t log_page)
1602 {
1603 NvmeAsyncEvent *event;
1604
1605 trace_pci_nvme_enqueue_event(event_type, event_info, log_page);
1606
1607 if (n->aer_queued == n->params.aer_max_queued) {
1608 trace_pci_nvme_enqueue_event_noqueue(n->aer_queued);
1609 return;
1610 }
1611
1612 event = g_new(NvmeAsyncEvent, 1);
1613 event->result = (NvmeAerResult) {
1614 .event_type = event_type,
1615 .event_info = event_info,
1616 .log_page = log_page,
1617 };
1618
1619 QTAILQ_INSERT_TAIL(&n->aer_queue, event, entry);
1620 n->aer_queued++;
1621
1622 nvme_process_aers(n);
1623 }
1624
1625 static void nvme_smart_event(NvmeCtrl *n, uint8_t event)
1626 {
1627 uint8_t aer_info;
1628
1629 /* Ref SPEC <Asynchronous Event Information 0x2013 SMART / Health Status> */
1630 if (!(NVME_AEC_SMART(n->features.async_config) & event)) {
1631 return;
1632 }
1633
1634 switch (event) {
1635 case NVME_SMART_SPARE:
1636 aer_info = NVME_AER_INFO_SMART_SPARE_THRESH;
1637 break;
1638 case NVME_SMART_TEMPERATURE:
1639 aer_info = NVME_AER_INFO_SMART_TEMP_THRESH;
1640 break;
1641 case NVME_SMART_RELIABILITY:
1642 case NVME_SMART_MEDIA_READ_ONLY:
1643 case NVME_SMART_FAILED_VOLATILE_MEDIA:
1644 case NVME_SMART_PMR_UNRELIABLE:
1645 aer_info = NVME_AER_INFO_SMART_RELIABILITY;
1646 break;
1647 default:
1648 return;
1649 }
1650
1651 nvme_enqueue_event(n, NVME_AER_TYPE_SMART, aer_info, NVME_LOG_SMART_INFO);
1652 }
1653
1654 static void nvme_clear_events(NvmeCtrl *n, uint8_t event_type)
1655 {
1656 n->aer_mask &= ~(1 << event_type);
1657 if (!QTAILQ_EMPTY(&n->aer_queue)) {
1658 nvme_process_aers(n);
1659 }
1660 }
1661
1662 static inline uint16_t nvme_check_mdts(NvmeCtrl *n, size_t len)
1663 {
1664 uint8_t mdts = n->params.mdts;
1665
1666 if (mdts && len > n->page_size << mdts) {
1667 trace_pci_nvme_err_mdts(len);
1668 return NVME_INVALID_FIELD | NVME_DNR;
1669 }
1670
1671 return NVME_SUCCESS;
1672 }
1673
1674 static inline uint16_t nvme_check_bounds(NvmeNamespace *ns, uint64_t slba,
1675 uint32_t nlb)
1676 {
1677 uint64_t nsze = le64_to_cpu(ns->id_ns.nsze);
1678
1679 if (unlikely(UINT64_MAX - slba < nlb || slba + nlb > nsze)) {
1680 trace_pci_nvme_err_invalid_lba_range(slba, nlb, nsze);
1681 return NVME_LBA_RANGE | NVME_DNR;
1682 }
1683
1684 return NVME_SUCCESS;
1685 }
1686
1687 static int nvme_block_status_all(NvmeNamespace *ns, uint64_t slba,
1688 uint32_t nlb, int flags)
1689 {
1690 BlockDriverState *bs = blk_bs(ns->blkconf.blk);
1691
1692 int64_t pnum = 0, bytes = nvme_l2b(ns, nlb);
1693 int64_t offset = nvme_l2b(ns, slba);
1694 int ret;
1695
1696 /*
1697 * `pnum` holds the number of bytes after offset that shares the same
1698 * allocation status as the byte at offset. If `pnum` is different from
1699 * `bytes`, we should check the allocation status of the next range and
1700 * continue this until all bytes have been checked.
1701 */
1702 do {
1703 bytes -= pnum;
1704
1705 ret = bdrv_block_status(bs, offset, bytes, &pnum, NULL, NULL);
1706 if (ret < 0) {
1707 return ret;
1708 }
1709
1710
1711 trace_pci_nvme_block_status(offset, bytes, pnum, ret,
1712 !!(ret & BDRV_BLOCK_ZERO));
1713
1714 if (!(ret & flags)) {
1715 return 1;
1716 }
1717
1718 offset += pnum;
1719 } while (pnum != bytes);
1720
1721 return 0;
1722 }
1723
1724 static uint16_t nvme_check_dulbe(NvmeNamespace *ns, uint64_t slba,
1725 uint32_t nlb)
1726 {
1727 int ret;
1728 Error *err = NULL;
1729
1730 ret = nvme_block_status_all(ns, slba, nlb, BDRV_BLOCK_DATA);
1731 if (ret) {
1732 if (ret < 0) {
1733 error_setg_errno(&err, -ret, "unable to get block status");
1734 error_report_err(err);
1735
1736 return NVME_INTERNAL_DEV_ERROR;
1737 }
1738
1739 return NVME_DULB;
1740 }
1741
1742 return NVME_SUCCESS;
1743 }
1744
1745 static void nvme_aio_err(NvmeRequest *req, int ret)
1746 {
1747 uint16_t status = NVME_SUCCESS;
1748 Error *local_err = NULL;
1749
1750 switch (req->cmd.opcode) {
1751 case NVME_CMD_READ:
1752 status = NVME_UNRECOVERED_READ;
1753 break;
1754 case NVME_CMD_FLUSH:
1755 case NVME_CMD_WRITE:
1756 case NVME_CMD_WRITE_ZEROES:
1757 case NVME_CMD_ZONE_APPEND:
1758 case NVME_CMD_COPY:
1759 status = NVME_WRITE_FAULT;
1760 break;
1761 default:
1762 status = NVME_INTERNAL_DEV_ERROR;
1763 break;
1764 }
1765
1766 trace_pci_nvme_err_aio(nvme_cid(req), strerror(-ret), status);
1767
1768 error_setg_errno(&local_err, -ret, "aio failed");
1769 error_report_err(local_err);
1770
1771 /*
1772 * Set the command status code to the first encountered error but allow a
1773 * subsequent Internal Device Error to trump it.
1774 */
1775 if (req->status && status != NVME_INTERNAL_DEV_ERROR) {
1776 return;
1777 }
1778
1779 req->status = status;
1780 }
1781
1782 static inline uint32_t nvme_zone_idx(NvmeNamespace *ns, uint64_t slba)
1783 {
1784 return ns->zone_size_log2 > 0 ? slba >> ns->zone_size_log2 :
1785 slba / ns->zone_size;
1786 }
1787
1788 static inline NvmeZone *nvme_get_zone_by_slba(NvmeNamespace *ns, uint64_t slba)
1789 {
1790 uint32_t zone_idx = nvme_zone_idx(ns, slba);
1791
1792 if (zone_idx >= ns->num_zones) {
1793 return NULL;
1794 }
1795
1796 return &ns->zone_array[zone_idx];
1797 }
1798
1799 static uint16_t nvme_check_zone_state_for_write(NvmeZone *zone)
1800 {
1801 uint64_t zslba = zone->d.zslba;
1802
1803 switch (nvme_get_zone_state(zone)) {
1804 case NVME_ZONE_STATE_EMPTY:
1805 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1806 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1807 case NVME_ZONE_STATE_CLOSED:
1808 return NVME_SUCCESS;
1809 case NVME_ZONE_STATE_FULL:
1810 trace_pci_nvme_err_zone_is_full(zslba);
1811 return NVME_ZONE_FULL;
1812 case NVME_ZONE_STATE_OFFLINE:
1813 trace_pci_nvme_err_zone_is_offline(zslba);
1814 return NVME_ZONE_OFFLINE;
1815 case NVME_ZONE_STATE_READ_ONLY:
1816 trace_pci_nvme_err_zone_is_read_only(zslba);
1817 return NVME_ZONE_READ_ONLY;
1818 default:
1819 assert(false);
1820 }
1821
1822 return NVME_INTERNAL_DEV_ERROR;
1823 }
1824
1825 static uint16_t nvme_check_zone_write(NvmeNamespace *ns, NvmeZone *zone,
1826 uint64_t slba, uint32_t nlb)
1827 {
1828 uint64_t zcap = nvme_zone_wr_boundary(zone);
1829 uint16_t status;
1830
1831 status = nvme_check_zone_state_for_write(zone);
1832 if (status) {
1833 return status;
1834 }
1835
1836 if (zone->d.za & NVME_ZA_ZRWA_VALID) {
1837 uint64_t ezrwa = zone->w_ptr + 2 * ns->zns.zrwas;
1838
1839 if (slba < zone->w_ptr || slba + nlb > ezrwa) {
1840 trace_pci_nvme_err_zone_invalid_write(slba, zone->w_ptr);
1841 return NVME_ZONE_INVALID_WRITE;
1842 }
1843 } else {
1844 if (unlikely(slba != zone->w_ptr)) {
1845 trace_pci_nvme_err_write_not_at_wp(slba, zone->d.zslba,
1846 zone->w_ptr);
1847 return NVME_ZONE_INVALID_WRITE;
1848 }
1849 }
1850
1851 if (unlikely((slba + nlb) > zcap)) {
1852 trace_pci_nvme_err_zone_boundary(slba, nlb, zcap);
1853 return NVME_ZONE_BOUNDARY_ERROR;
1854 }
1855
1856 return NVME_SUCCESS;
1857 }
1858
1859 static uint16_t nvme_check_zone_state_for_read(NvmeZone *zone)
1860 {
1861 switch (nvme_get_zone_state(zone)) {
1862 case NVME_ZONE_STATE_EMPTY:
1863 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1864 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1865 case NVME_ZONE_STATE_FULL:
1866 case NVME_ZONE_STATE_CLOSED:
1867 case NVME_ZONE_STATE_READ_ONLY:
1868 return NVME_SUCCESS;
1869 case NVME_ZONE_STATE_OFFLINE:
1870 trace_pci_nvme_err_zone_is_offline(zone->d.zslba);
1871 return NVME_ZONE_OFFLINE;
1872 default:
1873 assert(false);
1874 }
1875
1876 return NVME_INTERNAL_DEV_ERROR;
1877 }
1878
1879 static uint16_t nvme_check_zone_read(NvmeNamespace *ns, uint64_t slba,
1880 uint32_t nlb)
1881 {
1882 NvmeZone *zone;
1883 uint64_t bndry, end;
1884 uint16_t status;
1885
1886 zone = nvme_get_zone_by_slba(ns, slba);
1887 assert(zone);
1888
1889 bndry = nvme_zone_rd_boundary(ns, zone);
1890 end = slba + nlb;
1891
1892 status = nvme_check_zone_state_for_read(zone);
1893 if (status) {
1894 ;
1895 } else if (unlikely(end > bndry)) {
1896 if (!ns->params.cross_zone_read) {
1897 status = NVME_ZONE_BOUNDARY_ERROR;
1898 } else {
1899 /*
1900 * Read across zone boundary - check that all subsequent
1901 * zones that are being read have an appropriate state.
1902 */
1903 do {
1904 zone++;
1905 status = nvme_check_zone_state_for_read(zone);
1906 if (status) {
1907 break;
1908 }
1909 } while (end > nvme_zone_rd_boundary(ns, zone));
1910 }
1911 }
1912
1913 return status;
1914 }
1915
1916 static uint16_t nvme_zrm_finish(NvmeNamespace *ns, NvmeZone *zone)
1917 {
1918 switch (nvme_get_zone_state(zone)) {
1919 case NVME_ZONE_STATE_FULL:
1920 return NVME_SUCCESS;
1921
1922 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1923 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1924 nvme_aor_dec_open(ns);
1925 /* fallthrough */
1926 case NVME_ZONE_STATE_CLOSED:
1927 nvme_aor_dec_active(ns);
1928
1929 if (zone->d.za & NVME_ZA_ZRWA_VALID) {
1930 zone->d.za &= ~NVME_ZA_ZRWA_VALID;
1931 if (ns->params.numzrwa) {
1932 ns->zns.numzrwa++;
1933 }
1934 }
1935
1936 /* fallthrough */
1937 case NVME_ZONE_STATE_EMPTY:
1938 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_FULL);
1939 return NVME_SUCCESS;
1940
1941 default:
1942 return NVME_ZONE_INVAL_TRANSITION;
1943 }
1944 }
1945
1946 static uint16_t nvme_zrm_close(NvmeNamespace *ns, NvmeZone *zone)
1947 {
1948 switch (nvme_get_zone_state(zone)) {
1949 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1950 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1951 nvme_aor_dec_open(ns);
1952 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_CLOSED);
1953 /* fall through */
1954 case NVME_ZONE_STATE_CLOSED:
1955 return NVME_SUCCESS;
1956
1957 default:
1958 return NVME_ZONE_INVAL_TRANSITION;
1959 }
1960 }
1961
1962 static uint16_t nvme_zrm_reset(NvmeNamespace *ns, NvmeZone *zone)
1963 {
1964 switch (nvme_get_zone_state(zone)) {
1965 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1966 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1967 nvme_aor_dec_open(ns);
1968 /* fallthrough */
1969 case NVME_ZONE_STATE_CLOSED:
1970 nvme_aor_dec_active(ns);
1971
1972 if (zone->d.za & NVME_ZA_ZRWA_VALID) {
1973 if (ns->params.numzrwa) {
1974 ns->zns.numzrwa++;
1975 }
1976 }
1977
1978 /* fallthrough */
1979 case NVME_ZONE_STATE_FULL:
1980 zone->w_ptr = zone->d.zslba;
1981 zone->d.wp = zone->w_ptr;
1982 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_EMPTY);
1983 /* fallthrough */
1984 case NVME_ZONE_STATE_EMPTY:
1985 return NVME_SUCCESS;
1986
1987 default:
1988 return NVME_ZONE_INVAL_TRANSITION;
1989 }
1990 }
1991
1992 static void nvme_zrm_auto_transition_zone(NvmeNamespace *ns)
1993 {
1994 NvmeZone *zone;
1995
1996 if (ns->params.max_open_zones &&
1997 ns->nr_open_zones == ns->params.max_open_zones) {
1998 zone = QTAILQ_FIRST(&ns->imp_open_zones);
1999 if (zone) {
2000 /*
2001 * Automatically close this implicitly open zone.
2002 */
2003 QTAILQ_REMOVE(&ns->imp_open_zones, zone, entry);
2004 nvme_zrm_close(ns, zone);
2005 }
2006 }
2007 }
2008
2009 enum {
2010 NVME_ZRM_AUTO = 1 << 0,
2011 NVME_ZRM_ZRWA = 1 << 1,
2012 };
2013
2014 static uint16_t nvme_zrm_open_flags(NvmeCtrl *n, NvmeNamespace *ns,
2015 NvmeZone *zone, int flags)
2016 {
2017 int act = 0;
2018 uint16_t status;
2019
2020 switch (nvme_get_zone_state(zone)) {
2021 case NVME_ZONE_STATE_EMPTY:
2022 act = 1;
2023
2024 /* fallthrough */
2025
2026 case NVME_ZONE_STATE_CLOSED:
2027 if (n->params.auto_transition_zones) {
2028 nvme_zrm_auto_transition_zone(ns);
2029 }
2030 status = nvme_zns_check_resources(ns, act, 1,
2031 (flags & NVME_ZRM_ZRWA) ? 1 : 0);
2032 if (status) {
2033 return status;
2034 }
2035
2036 if (act) {
2037 nvme_aor_inc_active(ns);
2038 }
2039
2040 nvme_aor_inc_open(ns);
2041
2042 if (flags & NVME_ZRM_AUTO) {
2043 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_IMPLICITLY_OPEN);
2044 return NVME_SUCCESS;
2045 }
2046
2047 /* fallthrough */
2048
2049 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
2050 if (flags & NVME_ZRM_AUTO) {
2051 return NVME_SUCCESS;
2052 }
2053
2054 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_EXPLICITLY_OPEN);
2055
2056 /* fallthrough */
2057
2058 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
2059 if (flags & NVME_ZRM_ZRWA) {
2060 ns->zns.numzrwa--;
2061
2062 zone->d.za |= NVME_ZA_ZRWA_VALID;
2063 }
2064
2065 return NVME_SUCCESS;
2066
2067 default:
2068 return NVME_ZONE_INVAL_TRANSITION;
2069 }
2070 }
2071
2072 static inline uint16_t nvme_zrm_auto(NvmeCtrl *n, NvmeNamespace *ns,
2073 NvmeZone *zone)
2074 {
2075 return nvme_zrm_open_flags(n, ns, zone, NVME_ZRM_AUTO);
2076 }
2077
2078 static void nvme_advance_zone_wp(NvmeNamespace *ns, NvmeZone *zone,
2079 uint32_t nlb)
2080 {
2081 zone->d.wp += nlb;
2082
2083 if (zone->d.wp == nvme_zone_wr_boundary(zone)) {
2084 nvme_zrm_finish(ns, zone);
2085 }
2086 }
2087
2088 static void nvme_zoned_zrwa_implicit_flush(NvmeNamespace *ns, NvmeZone *zone,
2089 uint32_t nlbc)
2090 {
2091 uint16_t nzrwafgs = DIV_ROUND_UP(nlbc, ns->zns.zrwafg);
2092
2093 nlbc = nzrwafgs * ns->zns.zrwafg;
2094
2095 trace_pci_nvme_zoned_zrwa_implicit_flush(zone->d.zslba, nlbc);
2096
2097 zone->w_ptr += nlbc;
2098
2099 nvme_advance_zone_wp(ns, zone, nlbc);
2100 }
2101
2102 static void nvme_finalize_zoned_write(NvmeNamespace *ns, NvmeRequest *req)
2103 {
2104 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2105 NvmeZone *zone;
2106 uint64_t slba;
2107 uint32_t nlb;
2108
2109 slba = le64_to_cpu(rw->slba);
2110 nlb = le16_to_cpu(rw->nlb) + 1;
2111 zone = nvme_get_zone_by_slba(ns, slba);
2112 assert(zone);
2113
2114 if (zone->d.za & NVME_ZA_ZRWA_VALID) {
2115 uint64_t ezrwa = zone->w_ptr + ns->zns.zrwas - 1;
2116 uint64_t elba = slba + nlb - 1;
2117
2118 if (elba > ezrwa) {
2119 nvme_zoned_zrwa_implicit_flush(ns, zone, elba - ezrwa);
2120 }
2121
2122 return;
2123 }
2124
2125 nvme_advance_zone_wp(ns, zone, nlb);
2126 }
2127
2128 static inline bool nvme_is_write(NvmeRequest *req)
2129 {
2130 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2131
2132 return rw->opcode == NVME_CMD_WRITE ||
2133 rw->opcode == NVME_CMD_ZONE_APPEND ||
2134 rw->opcode == NVME_CMD_WRITE_ZEROES;
2135 }
2136
2137 static AioContext *nvme_get_aio_context(BlockAIOCB *acb)
2138 {
2139 return qemu_get_aio_context();
2140 }
2141
2142 static void nvme_misc_cb(void *opaque, int ret)
2143 {
2144 NvmeRequest *req = opaque;
2145
2146 trace_pci_nvme_misc_cb(nvme_cid(req));
2147
2148 if (ret) {
2149 nvme_aio_err(req, ret);
2150 }
2151
2152 nvme_enqueue_req_completion(nvme_cq(req), req);
2153 }
2154
2155 void nvme_rw_complete_cb(void *opaque, int ret)
2156 {
2157 NvmeRequest *req = opaque;
2158 NvmeNamespace *ns = req->ns;
2159 BlockBackend *blk = ns->blkconf.blk;
2160 BlockAcctCookie *acct = &req->acct;
2161 BlockAcctStats *stats = blk_get_stats(blk);
2162
2163 trace_pci_nvme_rw_complete_cb(nvme_cid(req), blk_name(blk));
2164
2165 if (ret) {
2166 block_acct_failed(stats, acct);
2167 nvme_aio_err(req, ret);
2168 } else {
2169 block_acct_done(stats, acct);
2170 }
2171
2172 if (ns->params.zoned && nvme_is_write(req)) {
2173 nvme_finalize_zoned_write(ns, req);
2174 }
2175
2176 nvme_enqueue_req_completion(nvme_cq(req), req);
2177 }
2178
2179 static void nvme_rw_cb(void *opaque, int ret)
2180 {
2181 NvmeRequest *req = opaque;
2182 NvmeNamespace *ns = req->ns;
2183
2184 BlockBackend *blk = ns->blkconf.blk;
2185
2186 trace_pci_nvme_rw_cb(nvme_cid(req), blk_name(blk));
2187
2188 if (ret) {
2189 goto out;
2190 }
2191
2192 if (ns->lbaf.ms) {
2193 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2194 uint64_t slba = le64_to_cpu(rw->slba);
2195 uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb) + 1;
2196 uint64_t offset = nvme_moff(ns, slba);
2197
2198 if (req->cmd.opcode == NVME_CMD_WRITE_ZEROES) {
2199 size_t mlen = nvme_m2b(ns, nlb);
2200
2201 req->aiocb = blk_aio_pwrite_zeroes(blk, offset, mlen,
2202 BDRV_REQ_MAY_UNMAP,
2203 nvme_rw_complete_cb, req);
2204 return;
2205 }
2206
2207 if (nvme_ns_ext(ns) || req->cmd.mptr) {
2208 uint16_t status;
2209
2210 nvme_sg_unmap(&req->sg);
2211 status = nvme_map_mdata(nvme_ctrl(req), nlb, req);
2212 if (status) {
2213 ret = -EFAULT;
2214 goto out;
2215 }
2216
2217 if (req->cmd.opcode == NVME_CMD_READ) {
2218 return nvme_blk_read(blk, offset, 1, nvme_rw_complete_cb, req);
2219 }
2220
2221 return nvme_blk_write(blk, offset, 1, nvme_rw_complete_cb, req);
2222 }
2223 }
2224
2225 out:
2226 nvme_rw_complete_cb(req, ret);
2227 }
2228
2229 static void nvme_verify_cb(void *opaque, int ret)
2230 {
2231 NvmeBounceContext *ctx = opaque;
2232 NvmeRequest *req = ctx->req;
2233 NvmeNamespace *ns = req->ns;
2234 BlockBackend *blk = ns->blkconf.blk;
2235 BlockAcctCookie *acct = &req->acct;
2236 BlockAcctStats *stats = blk_get_stats(blk);
2237 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2238 uint64_t slba = le64_to_cpu(rw->slba);
2239 uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
2240 uint16_t apptag = le16_to_cpu(rw->apptag);
2241 uint16_t appmask = le16_to_cpu(rw->appmask);
2242 uint64_t reftag = le32_to_cpu(rw->reftag);
2243 uint64_t cdw3 = le32_to_cpu(rw->cdw3);
2244 uint16_t status;
2245
2246 reftag |= cdw3 << 32;
2247
2248 trace_pci_nvme_verify_cb(nvme_cid(req), prinfo, apptag, appmask, reftag);
2249
2250 if (ret) {
2251 block_acct_failed(stats, acct);
2252 nvme_aio_err(req, ret);
2253 goto out;
2254 }
2255
2256 block_acct_done(stats, acct);
2257
2258 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
2259 status = nvme_dif_mangle_mdata(ns, ctx->mdata.bounce,
2260 ctx->mdata.iov.size, slba);
2261 if (status) {
2262 req->status = status;
2263 goto out;
2264 }
2265
2266 req->status = nvme_dif_check(ns, ctx->data.bounce, ctx->data.iov.size,
2267 ctx->mdata.bounce, ctx->mdata.iov.size,
2268 prinfo, slba, apptag, appmask, &reftag);
2269 }
2270
2271 out:
2272 qemu_iovec_destroy(&ctx->data.iov);
2273 g_free(ctx->data.bounce);
2274
2275 qemu_iovec_destroy(&ctx->mdata.iov);
2276 g_free(ctx->mdata.bounce);
2277
2278 g_free(ctx);
2279
2280 nvme_enqueue_req_completion(nvme_cq(req), req);
2281 }
2282
2283
2284 static void nvme_verify_mdata_in_cb(void *opaque, int ret)
2285 {
2286 NvmeBounceContext *ctx = opaque;
2287 NvmeRequest *req = ctx->req;
2288 NvmeNamespace *ns = req->ns;
2289 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2290 uint64_t slba = le64_to_cpu(rw->slba);
2291 uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
2292 size_t mlen = nvme_m2b(ns, nlb);
2293 uint64_t offset = nvme_moff(ns, slba);
2294 BlockBackend *blk = ns->blkconf.blk;
2295
2296 trace_pci_nvme_verify_mdata_in_cb(nvme_cid(req), blk_name(blk));
2297
2298 if (ret) {
2299 goto out;
2300 }
2301
2302 ctx->mdata.bounce = g_malloc(mlen);
2303
2304 qemu_iovec_reset(&ctx->mdata.iov);
2305 qemu_iovec_add(&ctx->mdata.iov, ctx->mdata.bounce, mlen);
2306
2307 req->aiocb = blk_aio_preadv(blk, offset, &ctx->mdata.iov, 0,
2308 nvme_verify_cb, ctx);
2309 return;
2310
2311 out:
2312 nvme_verify_cb(ctx, ret);
2313 }
2314
2315 struct nvme_compare_ctx {
2316 struct {
2317 QEMUIOVector iov;
2318 uint8_t *bounce;
2319 } data;
2320
2321 struct {
2322 QEMUIOVector iov;
2323 uint8_t *bounce;
2324 } mdata;
2325 };
2326
2327 static void nvme_compare_mdata_cb(void *opaque, int ret)
2328 {
2329 NvmeRequest *req = opaque;
2330 NvmeNamespace *ns = req->ns;
2331 NvmeCtrl *n = nvme_ctrl(req);
2332 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2333 uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
2334 uint16_t apptag = le16_to_cpu(rw->apptag);
2335 uint16_t appmask = le16_to_cpu(rw->appmask);
2336 uint64_t reftag = le32_to_cpu(rw->reftag);
2337 uint64_t cdw3 = le32_to_cpu(rw->cdw3);
2338 struct nvme_compare_ctx *ctx = req->opaque;
2339 g_autofree uint8_t *buf = NULL;
2340 BlockBackend *blk = ns->blkconf.blk;
2341 BlockAcctCookie *acct = &req->acct;
2342 BlockAcctStats *stats = blk_get_stats(blk);
2343 uint16_t status = NVME_SUCCESS;
2344
2345 reftag |= cdw3 << 32;
2346
2347 trace_pci_nvme_compare_mdata_cb(nvme_cid(req));
2348
2349 if (ret) {
2350 block_acct_failed(stats, acct);
2351 nvme_aio_err(req, ret);
2352 goto out;
2353 }
2354
2355 buf = g_malloc(ctx->mdata.iov.size);
2356
2357 status = nvme_bounce_mdata(n, buf, ctx->mdata.iov.size,
2358 NVME_TX_DIRECTION_TO_DEVICE, req);
2359 if (status) {
2360 req->status = status;
2361 goto out;
2362 }
2363
2364 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
2365 uint64_t slba = le64_to_cpu(rw->slba);
2366 uint8_t *bufp;
2367 uint8_t *mbufp = ctx->mdata.bounce;
2368 uint8_t *end = mbufp + ctx->mdata.iov.size;
2369 int16_t pil = 0;
2370
2371 status = nvme_dif_check(ns, ctx->data.bounce, ctx->data.iov.size,
2372 ctx->mdata.bounce, ctx->mdata.iov.size, prinfo,
2373 slba, apptag, appmask, &reftag);
2374 if (status) {
2375 req->status = status;
2376 goto out;
2377 }
2378
2379 /*
2380 * When formatted with protection information, do not compare the DIF
2381 * tuple.
2382 */
2383 if (!(ns->id_ns.dps & NVME_ID_NS_DPS_FIRST_EIGHT)) {
2384 pil = ns->lbaf.ms - nvme_pi_tuple_size(ns);
2385 }
2386
2387 for (bufp = buf; mbufp < end; bufp += ns->lbaf.ms, mbufp += ns->lbaf.ms) {
2388 if (memcmp(bufp + pil, mbufp + pil, ns->lbaf.ms - pil)) {
2389 req->status = NVME_CMP_FAILURE | NVME_DNR;
2390 goto out;
2391 }
2392 }
2393
2394 goto out;
2395 }
2396
2397 if (memcmp(buf, ctx->mdata.bounce, ctx->mdata.iov.size)) {
2398 req->status = NVME_CMP_FAILURE | NVME_DNR;
2399 goto out;
2400 }
2401
2402 block_acct_done(stats, acct);
2403
2404 out:
2405 qemu_iovec_destroy(&ctx->data.iov);
2406 g_free(ctx->data.bounce);
2407
2408 qemu_iovec_destroy(&ctx->mdata.iov);
2409 g_free(ctx->mdata.bounce);
2410
2411 g_free(ctx);
2412
2413 nvme_enqueue_req_completion(nvme_cq(req), req);
2414 }
2415
2416 static void nvme_compare_data_cb(void *opaque, int ret)
2417 {
2418 NvmeRequest *req = opaque;
2419 NvmeCtrl *n = nvme_ctrl(req);
2420 NvmeNamespace *ns = req->ns;
2421 BlockBackend *blk = ns->blkconf.blk;
2422 BlockAcctCookie *acct = &req->acct;
2423 BlockAcctStats *stats = blk_get_stats(blk);
2424
2425 struct nvme_compare_ctx *ctx = req->opaque;
2426 g_autofree uint8_t *buf = NULL;
2427 uint16_t status;
2428
2429 trace_pci_nvme_compare_data_cb(nvme_cid(req));
2430
2431 if (ret) {
2432 block_acct_failed(stats, acct);
2433 nvme_aio_err(req, ret);
2434 goto out;
2435 }
2436
2437 buf = g_malloc(ctx->data.iov.size);
2438
2439 status = nvme_bounce_data(n, buf, ctx->data.iov.size,
2440 NVME_TX_DIRECTION_TO_DEVICE, req);
2441 if (status) {
2442 req->status = status;
2443 goto out;
2444 }
2445
2446 if (memcmp(buf, ctx->data.bounce, ctx->data.iov.size)) {
2447 req->status = NVME_CMP_FAILURE | NVME_DNR;
2448 goto out;
2449 }
2450
2451 if (ns->lbaf.ms) {
2452 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2453 uint64_t slba = le64_to_cpu(rw->slba);
2454 uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
2455 size_t mlen = nvme_m2b(ns, nlb);
2456 uint64_t offset = nvme_moff(ns, slba);
2457
2458 ctx->mdata.bounce = g_malloc(mlen);
2459
2460 qemu_iovec_init(&ctx->mdata.iov, 1);
2461 qemu_iovec_add(&ctx->mdata.iov, ctx->mdata.bounce, mlen);
2462
2463 req->aiocb = blk_aio_preadv(blk, offset, &ctx->mdata.iov, 0,
2464 nvme_compare_mdata_cb, req);
2465 return;
2466 }
2467
2468 block_acct_done(stats, acct);
2469
2470 out:
2471 qemu_iovec_destroy(&ctx->data.iov);
2472 g_free(ctx->data.bounce);
2473 g_free(ctx);
2474
2475 nvme_enqueue_req_completion(nvme_cq(req), req);
2476 }
2477
2478 typedef struct NvmeDSMAIOCB {
2479 BlockAIOCB common;
2480 BlockAIOCB *aiocb;
2481 NvmeRequest *req;
2482 int ret;
2483
2484 NvmeDsmRange *range;
2485 unsigned int nr;
2486 unsigned int idx;
2487 } NvmeDSMAIOCB;
2488
2489 static void nvme_dsm_cancel(BlockAIOCB *aiocb)
2490 {
2491 NvmeDSMAIOCB *iocb = container_of(aiocb, NvmeDSMAIOCB, common);
2492
2493 /* break nvme_dsm_cb loop */
2494 iocb->idx = iocb->nr;
2495 iocb->ret = -ECANCELED;
2496
2497 if (iocb->aiocb) {
2498 blk_aio_cancel_async(iocb->aiocb);
2499 iocb->aiocb = NULL;
2500 } else {
2501 /*
2502 * We only reach this if nvme_dsm_cancel() has already been called or
2503 * the command ran to completion.
2504 */
2505 assert(iocb->idx == iocb->nr);
2506 }
2507 }
2508
2509 static const AIOCBInfo nvme_dsm_aiocb_info = {
2510 .aiocb_size = sizeof(NvmeDSMAIOCB),
2511 .cancel_async = nvme_dsm_cancel,
2512 };
2513
2514 static void nvme_dsm_cb(void *opaque, int ret);
2515
2516 static void nvme_dsm_md_cb(void *opaque, int ret)
2517 {
2518 NvmeDSMAIOCB *iocb = opaque;
2519 NvmeRequest *req = iocb->req;
2520 NvmeNamespace *ns = req->ns;
2521 NvmeDsmRange *range;
2522 uint64_t slba;
2523 uint32_t nlb;
2524
2525 if (ret < 0 || iocb->ret < 0 || !ns->lbaf.ms) {
2526 goto done;
2527 }
2528
2529 range = &iocb->range[iocb->idx - 1];
2530 slba = le64_to_cpu(range->slba);
2531 nlb = le32_to_cpu(range->nlb);
2532
2533 /*
2534 * Check that all block were discarded (zeroed); otherwise we do not zero
2535 * the metadata.
2536 */
2537
2538 ret = nvme_block_status_all(ns, slba, nlb, BDRV_BLOCK_ZERO);
2539 if (ret) {
2540 if (ret < 0) {
2541 goto done;
2542 }
2543
2544 nvme_dsm_cb(iocb, 0);
2545 return;
2546 }
2547
2548 iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk, nvme_moff(ns, slba),
2549 nvme_m2b(ns, nlb), BDRV_REQ_MAY_UNMAP,
2550 nvme_dsm_cb, iocb);
2551 return;
2552
2553 done:
2554 nvme_dsm_cb(iocb, ret);
2555 }
2556
2557 static void nvme_dsm_cb(void *opaque, int ret)
2558 {
2559 NvmeDSMAIOCB *iocb = opaque;
2560 NvmeRequest *req = iocb->req;
2561 NvmeCtrl *n = nvme_ctrl(req);
2562 NvmeNamespace *ns = req->ns;
2563 NvmeDsmRange *range;
2564 uint64_t slba;
2565 uint32_t nlb;
2566
2567 if (iocb->ret < 0) {
2568 goto done;
2569 } else if (ret < 0) {
2570 iocb->ret = ret;
2571 goto done;
2572 }
2573
2574 next:
2575 if (iocb->idx == iocb->nr) {
2576 goto done;
2577 }
2578
2579 range = &iocb->range[iocb->idx++];
2580 slba = le64_to_cpu(range->slba);
2581 nlb = le32_to_cpu(range->nlb);
2582
2583 trace_pci_nvme_dsm_deallocate(slba, nlb);
2584
2585 if (nlb > n->dmrsl) {
2586 trace_pci_nvme_dsm_single_range_limit_exceeded(nlb, n->dmrsl);
2587 goto next;
2588 }
2589
2590 if (nvme_check_bounds(ns, slba, nlb)) {
2591 trace_pci_nvme_err_invalid_lba_range(slba, nlb,
2592 ns->id_ns.nsze);
2593 goto next;
2594 }
2595
2596 iocb->aiocb = blk_aio_pdiscard(ns->blkconf.blk, nvme_l2b(ns, slba),
2597 nvme_l2b(ns, nlb),
2598 nvme_dsm_md_cb, iocb);
2599 return;
2600
2601 done:
2602 iocb->aiocb = NULL;
2603 iocb->common.cb(iocb->common.opaque, iocb->ret);
2604 qemu_aio_unref(iocb);
2605 }
2606
2607 static uint16_t nvme_dsm(NvmeCtrl *n, NvmeRequest *req)
2608 {
2609 NvmeNamespace *ns = req->ns;
2610 NvmeDsmCmd *dsm = (NvmeDsmCmd *) &req->cmd;
2611 uint32_t attr = le32_to_cpu(dsm->attributes);
2612 uint32_t nr = (le32_to_cpu(dsm->nr) & 0xff) + 1;
2613 uint16_t status = NVME_SUCCESS;
2614
2615 trace_pci_nvme_dsm(nr, attr);
2616
2617 if (attr & NVME_DSMGMT_AD) {
2618 NvmeDSMAIOCB *iocb = blk_aio_get(&nvme_dsm_aiocb_info, ns->blkconf.blk,
2619 nvme_misc_cb, req);
2620
2621 iocb->req = req;
2622 iocb->ret = 0;
2623 iocb->range = g_new(NvmeDsmRange, nr);
2624 iocb->nr = nr;
2625 iocb->idx = 0;
2626
2627 status = nvme_h2c(n, (uint8_t *)iocb->range, sizeof(NvmeDsmRange) * nr,
2628 req);
2629 if (status) {
2630 g_free(iocb->range);
2631 qemu_aio_unref(iocb);
2632
2633 return status;
2634 }
2635
2636 req->aiocb = &iocb->common;
2637 nvme_dsm_cb(iocb, 0);
2638
2639 return NVME_NO_COMPLETE;
2640 }
2641
2642 return status;
2643 }
2644
2645 static uint16_t nvme_verify(NvmeCtrl *n, NvmeRequest *req)
2646 {
2647 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2648 NvmeNamespace *ns = req->ns;
2649 BlockBackend *blk = ns->blkconf.blk;
2650 uint64_t slba = le64_to_cpu(rw->slba);
2651 uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
2652 size_t len = nvme_l2b(ns, nlb);
2653 int64_t offset = nvme_l2b(ns, slba);
2654 uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
2655 uint32_t reftag = le32_to_cpu(rw->reftag);
2656 NvmeBounceContext *ctx = NULL;
2657 uint16_t status;
2658
2659 trace_pci_nvme_verify(nvme_cid(req), nvme_nsid(ns), slba, nlb);
2660
2661 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
2662 status = nvme_check_prinfo(ns, prinfo, slba, reftag);
2663 if (status) {
2664 return status;
2665 }
2666
2667 if (prinfo & NVME_PRINFO_PRACT) {
2668 return NVME_INVALID_PROT_INFO | NVME_DNR;
2669 }
2670 }
2671
2672 if (len > n->page_size << n->params.vsl) {
2673 return NVME_INVALID_FIELD | NVME_DNR;
2674 }
2675
2676 status = nvme_check_bounds(ns, slba, nlb);
2677 if (status) {
2678 return status;
2679 }
2680
2681 if (NVME_ERR_REC_DULBE(ns->features.err_rec)) {
2682 status = nvme_check_dulbe(ns, slba, nlb);
2683 if (status) {
2684 return status;
2685 }
2686 }
2687
2688 ctx = g_new0(NvmeBounceContext, 1);
2689 ctx->req = req;
2690
2691 ctx->data.bounce = g_malloc(len);
2692
2693 qemu_iovec_init(&ctx->data.iov, 1);
2694 qemu_iovec_add(&ctx->data.iov, ctx->data.bounce, len);
2695
2696 block_acct_start(blk_get_stats(blk), &req->acct, ctx->data.iov.size,
2697 BLOCK_ACCT_READ);
2698
2699 req->aiocb = blk_aio_preadv(ns->blkconf.blk, offset, &ctx->data.iov, 0,
2700 nvme_verify_mdata_in_cb, ctx);
2701 return NVME_NO_COMPLETE;
2702 }
2703
2704 typedef struct NvmeCopyAIOCB {
2705 BlockAIOCB common;
2706 BlockAIOCB *aiocb;
2707 NvmeRequest *req;
2708 int ret;
2709
2710 void *ranges;
2711 unsigned int format;
2712 int nr;
2713 int idx;
2714
2715 uint8_t *bounce;
2716 QEMUIOVector iov;
2717 struct {
2718 BlockAcctCookie read;
2719 BlockAcctCookie write;
2720 } acct;
2721
2722 uint64_t reftag;
2723 uint64_t slba;
2724
2725 NvmeZone *zone;
2726 } NvmeCopyAIOCB;
2727
2728 static void nvme_copy_cancel(BlockAIOCB *aiocb)
2729 {
2730 NvmeCopyAIOCB *iocb = container_of(aiocb, NvmeCopyAIOCB, common);
2731
2732 iocb->ret = -ECANCELED;
2733
2734 if (iocb->aiocb) {
2735 blk_aio_cancel_async(iocb->aiocb);
2736 iocb->aiocb = NULL;
2737 }
2738 }
2739
2740 static const AIOCBInfo nvme_copy_aiocb_info = {
2741 .aiocb_size = sizeof(NvmeCopyAIOCB),
2742 .cancel_async = nvme_copy_cancel,
2743 };
2744
2745 static void nvme_copy_done(NvmeCopyAIOCB *iocb)
2746 {
2747 NvmeRequest *req = iocb->req;
2748 NvmeNamespace *ns = req->ns;
2749 BlockAcctStats *stats = blk_get_stats(ns->blkconf.blk);
2750
2751 if (iocb->idx != iocb->nr) {
2752 req->cqe.result = cpu_to_le32(iocb->idx);
2753 }
2754
2755 qemu_iovec_destroy(&iocb->iov);
2756 g_free(iocb->bounce);
2757
2758 if (iocb->ret < 0) {
2759 block_acct_failed(stats, &iocb->acct.read);
2760 block_acct_failed(stats, &iocb->acct.write);
2761 } else {
2762 block_acct_done(stats, &iocb->acct.read);
2763 block_acct_done(stats, &iocb->acct.write);
2764 }
2765
2766 iocb->common.cb(iocb->common.opaque, iocb->ret);
2767 qemu_aio_unref(iocb);
2768 }
2769
2770 static void nvme_do_copy(NvmeCopyAIOCB *iocb);
2771
2772 static void nvme_copy_source_range_parse_format0(void *ranges, int idx,
2773 uint64_t *slba, uint32_t *nlb,
2774 uint16_t *apptag,
2775 uint16_t *appmask,
2776 uint64_t *reftag)
2777 {
2778 NvmeCopySourceRangeFormat0 *_ranges = ranges;
2779
2780 if (slba) {
2781 *slba = le64_to_cpu(_ranges[idx].slba);
2782 }
2783
2784 if (nlb) {
2785 *nlb = le16_to_cpu(_ranges[idx].nlb) + 1;
2786 }
2787
2788 if (apptag) {
2789 *apptag = le16_to_cpu(_ranges[idx].apptag);
2790 }
2791
2792 if (appmask) {
2793 *appmask = le16_to_cpu(_ranges[idx].appmask);
2794 }
2795
2796 if (reftag) {
2797 *reftag = le32_to_cpu(_ranges[idx].reftag);
2798 }
2799 }
2800
2801 static void nvme_copy_source_range_parse_format1(void *ranges, int idx,
2802 uint64_t *slba, uint32_t *nlb,
2803 uint16_t *apptag,
2804 uint16_t *appmask,
2805 uint64_t *reftag)
2806 {
2807 NvmeCopySourceRangeFormat1 *_ranges = ranges;
2808
2809 if (slba) {
2810 *slba = le64_to_cpu(_ranges[idx].slba);
2811 }
2812
2813 if (nlb) {
2814 *nlb = le16_to_cpu(_ranges[idx].nlb) + 1;
2815 }
2816
2817 if (apptag) {
2818 *apptag = le16_to_cpu(_ranges[idx].apptag);
2819 }
2820
2821 if (appmask) {
2822 *appmask = le16_to_cpu(_ranges[idx].appmask);
2823 }
2824
2825 if (reftag) {
2826 *reftag = 0;
2827
2828 *reftag |= (uint64_t)_ranges[idx].sr[4] << 40;
2829 *reftag |= (uint64_t)_ranges[idx].sr[5] << 32;
2830 *reftag |= (uint64_t)_ranges[idx].sr[6] << 24;
2831 *reftag |= (uint64_t)_ranges[idx].sr[7] << 16;
2832 *reftag |= (uint64_t)_ranges[idx].sr[8] << 8;
2833 *reftag |= (uint64_t)_ranges[idx].sr[9];
2834 }
2835 }
2836
2837 static void nvme_copy_source_range_parse(void *ranges, int idx, uint8_t format,
2838 uint64_t *slba, uint32_t *nlb,
2839 uint16_t *apptag, uint16_t *appmask,
2840 uint64_t *reftag)
2841 {
2842 switch (format) {
2843 case NVME_COPY_FORMAT_0:
2844 nvme_copy_source_range_parse_format0(ranges, idx, slba, nlb, apptag,
2845 appmask, reftag);
2846 break;
2847
2848 case NVME_COPY_FORMAT_1:
2849 nvme_copy_source_range_parse_format1(ranges, idx, slba, nlb, apptag,
2850 appmask, reftag);
2851 break;
2852
2853 default:
2854 abort();
2855 }
2856 }
2857
2858 static void nvme_copy_out_completed_cb(void *opaque, int ret)
2859 {
2860 NvmeCopyAIOCB *iocb = opaque;
2861 NvmeRequest *req = iocb->req;
2862 NvmeNamespace *ns = req->ns;
2863 uint32_t nlb;
2864
2865 nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, NULL,
2866 &nlb, NULL, NULL, NULL);
2867
2868 if (ret < 0) {
2869 iocb->ret = ret;
2870 goto out;
2871 } else if (iocb->ret < 0) {
2872 goto out;
2873 }
2874
2875 if (ns->params.zoned) {
2876 nvme_advance_zone_wp(ns, iocb->zone, nlb);
2877 }
2878
2879 iocb->idx++;
2880 iocb->slba += nlb;
2881 out:
2882 nvme_do_copy(iocb);
2883 }
2884
2885 static void nvme_copy_out_cb(void *opaque, int ret)
2886 {
2887 NvmeCopyAIOCB *iocb = opaque;
2888 NvmeRequest *req = iocb->req;
2889 NvmeNamespace *ns = req->ns;
2890 uint32_t nlb;
2891 size_t mlen;
2892 uint8_t *mbounce;
2893
2894 if (ret < 0 || iocb->ret < 0 || !ns->lbaf.ms) {
2895 goto out;
2896 }
2897
2898 nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, NULL,
2899 &nlb, NULL, NULL, NULL);
2900
2901 mlen = nvme_m2b(ns, nlb);
2902 mbounce = iocb->bounce + nvme_l2b(ns, nlb);
2903
2904 qemu_iovec_reset(&iocb->iov);
2905 qemu_iovec_add(&iocb->iov, mbounce, mlen);
2906
2907 iocb->aiocb = blk_aio_pwritev(ns->blkconf.blk, nvme_moff(ns, iocb->slba),
2908 &iocb->iov, 0, nvme_copy_out_completed_cb,
2909 iocb);
2910
2911 return;
2912
2913 out:
2914 nvme_copy_out_completed_cb(iocb, ret);
2915 }
2916
2917 static void nvme_copy_in_completed_cb(void *opaque, int ret)
2918 {
2919 NvmeCopyAIOCB *iocb = opaque;
2920 NvmeRequest *req = iocb->req;
2921 NvmeNamespace *ns = req->ns;
2922 uint32_t nlb;
2923 uint64_t slba;
2924 uint16_t apptag, appmask;
2925 uint64_t reftag;
2926 size_t len;
2927 uint16_t status;
2928
2929 if (ret < 0) {
2930 iocb->ret = ret;
2931 goto out;
2932 } else if (iocb->ret < 0) {
2933 goto out;
2934 }
2935
2936 nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, &slba,
2937 &nlb, &apptag, &appmask, &reftag);
2938 len = nvme_l2b(ns, nlb);
2939
2940 trace_pci_nvme_copy_out(iocb->slba, nlb);
2941
2942 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
2943 NvmeCopyCmd *copy = (NvmeCopyCmd *)&req->cmd;
2944
2945 uint16_t prinfor = ((copy->control[0] >> 4) & 0xf);
2946 uint16_t prinfow = ((copy->control[2] >> 2) & 0xf);
2947
2948 size_t mlen = nvme_m2b(ns, nlb);
2949 uint8_t *mbounce = iocb->bounce + nvme_l2b(ns, nlb);
2950
2951 status = nvme_dif_mangle_mdata(ns, mbounce, mlen, slba);
2952 if (status) {
2953 goto invalid;
2954 }
2955 status = nvme_dif_check(ns, iocb->bounce, len, mbounce, mlen, prinfor,
2956 slba, apptag, appmask, &reftag);
2957 if (status) {
2958 goto invalid;
2959 }
2960
2961 apptag = le16_to_cpu(copy->apptag);
2962 appmask = le16_to_cpu(copy->appmask);
2963
2964 if (prinfow & NVME_PRINFO_PRACT) {
2965 status = nvme_check_prinfo(ns, prinfow, iocb->slba, iocb->reftag);
2966 if (status) {
2967 goto invalid;
2968 }
2969
2970 nvme_dif_pract_generate_dif(ns, iocb->bounce, len, mbounce, mlen,
2971 apptag, &iocb->reftag);
2972 } else {
2973 status = nvme_dif_check(ns, iocb->bounce, len, mbounce, mlen,
2974 prinfow, iocb->slba, apptag, appmask,
2975 &iocb->reftag);
2976 if (status) {
2977 goto invalid;
2978 }
2979 }
2980 }
2981
2982 status = nvme_check_bounds(ns, iocb->slba, nlb);
2983 if (status) {
2984 goto invalid;
2985 }
2986
2987 if (ns->params.zoned) {
2988 status = nvme_check_zone_write(ns, iocb->zone, iocb->slba, nlb);
2989 if (status) {
2990 goto invalid;
2991 }
2992
2993 if (!(iocb->zone->d.za & NVME_ZA_ZRWA_VALID)) {
2994 iocb->zone->w_ptr += nlb;
2995 }
2996 }
2997
2998 qemu_iovec_reset(&iocb->iov);
2999 qemu_iovec_add(&iocb->iov, iocb->bounce, len);
3000
3001 iocb->aiocb = blk_aio_pwritev(ns->blkconf.blk, nvme_l2b(ns, iocb->slba),
3002 &iocb->iov, 0, nvme_copy_out_cb, iocb);
3003
3004 return;
3005
3006 invalid:
3007 req->status = status;
3008 iocb->ret = -1;
3009 out:
3010 nvme_do_copy(iocb);
3011 }
3012
3013 static void nvme_copy_in_cb(void *opaque, int ret)
3014 {
3015 NvmeCopyAIOCB *iocb = opaque;
3016 NvmeRequest *req = iocb->req;
3017 NvmeNamespace *ns = req->ns;
3018 uint64_t slba;
3019 uint32_t nlb;
3020
3021 if (ret < 0 || iocb->ret < 0 || !ns->lbaf.ms) {
3022 goto out;
3023 }
3024
3025 nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, &slba,
3026 &nlb, NULL, NULL, NULL);
3027
3028 qemu_iovec_reset(&iocb->iov);
3029 qemu_iovec_add(&iocb->iov, iocb->bounce + nvme_l2b(ns, nlb),
3030 nvme_m2b(ns, nlb));
3031
3032 iocb->aiocb = blk_aio_preadv(ns->blkconf.blk, nvme_moff(ns, slba),
3033 &iocb->iov, 0, nvme_copy_in_completed_cb,
3034 iocb);
3035 return;
3036
3037 out:
3038 nvme_copy_in_completed_cb(iocb, ret);
3039 }
3040
3041 static void nvme_do_copy(NvmeCopyAIOCB *iocb)
3042 {
3043 NvmeRequest *req = iocb->req;
3044 NvmeNamespace *ns = req->ns;
3045 uint64_t slba;
3046 uint32_t nlb;
3047 size_t len;
3048 uint16_t status;
3049
3050 if (iocb->ret < 0) {
3051 goto done;
3052 }
3053
3054 if (iocb->idx == iocb->nr) {
3055 goto done;
3056 }
3057
3058 nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, &slba,
3059 &nlb, NULL, NULL, NULL);
3060 len = nvme_l2b(ns, nlb);
3061
3062 trace_pci_nvme_copy_source_range(slba, nlb);
3063
3064 if (nlb > le16_to_cpu(ns->id_ns.mssrl)) {
3065 status = NVME_CMD_SIZE_LIMIT | NVME_DNR;
3066 goto invalid;
3067 }
3068
3069 status = nvme_check_bounds(ns, slba, nlb);
3070 if (status) {
3071 goto invalid;
3072 }
3073
3074 if (NVME_ERR_REC_DULBE(ns->features.err_rec)) {
3075 status = nvme_check_dulbe(ns, slba, nlb);
3076 if (status) {
3077 goto invalid;
3078 }
3079 }
3080
3081 if (ns->params.zoned) {
3082 status = nvme_check_zone_read(ns, slba, nlb);
3083 if (status) {
3084 goto invalid;
3085 }
3086 }
3087
3088 qemu_iovec_reset(&iocb->iov);
3089 qemu_iovec_add(&iocb->iov, iocb->bounce, len);
3090
3091 iocb->aiocb = blk_aio_preadv(ns->blkconf.blk, nvme_l2b(ns, slba),
3092 &iocb->iov, 0, nvme_copy_in_cb, iocb);
3093 return;
3094
3095 invalid:
3096 req->status = status;
3097 iocb->ret = -1;
3098 done:
3099 nvme_copy_done(iocb);
3100 }
3101
3102 static uint16_t nvme_copy(NvmeCtrl *n, NvmeRequest *req)
3103 {
3104 NvmeNamespace *ns = req->ns;
3105 NvmeCopyCmd *copy = (NvmeCopyCmd *)&req->cmd;
3106 NvmeCopyAIOCB *iocb = blk_aio_get(&nvme_copy_aiocb_info, ns->blkconf.blk,
3107 nvme_misc_cb, req);
3108 uint16_t nr = copy->nr + 1;
3109 uint8_t format = copy->control[0] & 0xf;
3110 uint16_t prinfor = ((copy->control[0] >> 4) & 0xf);
3111 uint16_t prinfow = ((copy->control[2] >> 2) & 0xf);
3112 size_t len = sizeof(NvmeCopySourceRangeFormat0);
3113
3114 uint16_t status;
3115
3116 trace_pci_nvme_copy(nvme_cid(req), nvme_nsid(ns), nr, format);
3117
3118 iocb->ranges = NULL;
3119 iocb->zone = NULL;
3120
3121 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) &&
3122 ((prinfor & NVME_PRINFO_PRACT) != (prinfow & NVME_PRINFO_PRACT))) {
3123 status = NVME_INVALID_FIELD | NVME_DNR;
3124 goto invalid;
3125 }
3126
3127 if (!(n->id_ctrl.ocfs & (1 << format))) {
3128 trace_pci_nvme_err_copy_invalid_format(format);
3129 status = NVME_INVALID_FIELD | NVME_DNR;
3130 goto invalid;
3131 }
3132
3133 if (nr > ns->id_ns.msrc + 1) {
3134 status = NVME_CMD_SIZE_LIMIT | NVME_DNR;
3135 goto invalid;
3136 }
3137
3138 if ((ns->pif == 0x0 && format != 0x0) ||
3139 (ns->pif != 0x0 && format != 0x1)) {
3140 status = NVME_INVALID_FORMAT | NVME_DNR;
3141 goto invalid;
3142 }
3143
3144 if (ns->pif) {
3145 len = sizeof(NvmeCopySourceRangeFormat1);
3146 }
3147
3148 iocb->format = format;
3149 iocb->ranges = g_malloc_n(nr, len);
3150 status = nvme_h2c(n, (uint8_t *)iocb->ranges, len * nr, req);
3151 if (status) {
3152 goto invalid;
3153 }
3154
3155 iocb->slba = le64_to_cpu(copy->sdlba);
3156
3157 if (ns->params.zoned) {
3158 iocb->zone = nvme_get_zone_by_slba(ns, iocb->slba);
3159 if (!iocb->zone) {
3160 status = NVME_LBA_RANGE | NVME_DNR;
3161 goto invalid;
3162 }
3163
3164 status = nvme_zrm_auto(n, ns, iocb->zone);
3165 if (status) {
3166 goto invalid;
3167 }
3168 }
3169
3170 iocb->req = req;
3171 iocb->ret = 0;
3172 iocb->nr = nr;
3173 iocb->idx = 0;
3174 iocb->reftag = le32_to_cpu(copy->reftag);
3175 iocb->reftag |= (uint64_t)le32_to_cpu(copy->cdw3) << 32;
3176 iocb->bounce = g_malloc_n(le16_to_cpu(ns->id_ns.mssrl),
3177 ns->lbasz + ns->lbaf.ms);
3178
3179 qemu_iovec_init(&iocb->iov, 1);
3180
3181 block_acct_start(blk_get_stats(ns->blkconf.blk), &iocb->acct.read, 0,
3182 BLOCK_ACCT_READ);
3183 block_acct_start(blk_get_stats(ns->blkconf.blk), &iocb->acct.write, 0,
3184 BLOCK_ACCT_WRITE);
3185
3186 req->aiocb = &iocb->common;
3187 nvme_do_copy(iocb);
3188
3189 return NVME_NO_COMPLETE;
3190
3191 invalid:
3192 g_free(iocb->ranges);
3193 qemu_aio_unref(iocb);
3194 return status;
3195 }
3196
3197 static uint16_t nvme_compare(NvmeCtrl *n, NvmeRequest *req)
3198 {
3199 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
3200 NvmeNamespace *ns = req->ns;
3201 BlockBackend *blk = ns->blkconf.blk;
3202 uint64_t slba = le64_to_cpu(rw->slba);
3203 uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
3204 uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
3205 size_t data_len = nvme_l2b(ns, nlb);
3206 size_t len = data_len;
3207 int64_t offset = nvme_l2b(ns, slba);
3208 struct nvme_compare_ctx *ctx = NULL;
3209 uint16_t status;
3210
3211 trace_pci_nvme_compare(nvme_cid(req), nvme_nsid(ns), slba, nlb);
3212
3213 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) && (prinfo & NVME_PRINFO_PRACT)) {
3214 return NVME_INVALID_PROT_INFO | NVME_DNR;
3215 }
3216
3217 if (nvme_ns_ext(ns)) {
3218 len += nvme_m2b(ns, nlb);
3219 }
3220
3221 status = nvme_check_mdts(n, len);
3222 if (status) {
3223 return status;
3224 }
3225
3226 status = nvme_check_bounds(ns, slba, nlb);
3227 if (status) {
3228 return status;
3229 }
3230
3231 if (NVME_ERR_REC_DULBE(ns->features.err_rec)) {
3232 status = nvme_check_dulbe(ns, slba, nlb);
3233 if (status) {
3234 return status;
3235 }
3236 }
3237
3238 status = nvme_map_dptr(n, &req->sg, len, &req->cmd);
3239 if (status) {
3240 return status;
3241 }
3242
3243 ctx = g_new(struct nvme_compare_ctx, 1);
3244 ctx->data.bounce = g_malloc(data_len);
3245
3246 req->opaque = ctx;
3247
3248 qemu_iovec_init(&ctx->data.iov, 1);
3249 qemu_iovec_add(&ctx->data.iov, ctx->data.bounce, data_len);
3250
3251 block_acct_start(blk_get_stats(blk), &req->acct, data_len,
3252 BLOCK_ACCT_READ);
3253 req->aiocb = blk_aio_preadv(blk, offset, &ctx->data.iov, 0,
3254 nvme_compare_data_cb, req);
3255
3256 return NVME_NO_COMPLETE;
3257 }
3258
3259 typedef struct NvmeFlushAIOCB {
3260 BlockAIOCB common;
3261 BlockAIOCB *aiocb;
3262 NvmeRequest *req;
3263 int ret;
3264
3265 NvmeNamespace *ns;
3266 uint32_t nsid;
3267 bool broadcast;
3268 } NvmeFlushAIOCB;
3269
3270 static void nvme_flush_cancel(BlockAIOCB *acb)
3271 {
3272 NvmeFlushAIOCB *iocb = container_of(acb, NvmeFlushAIOCB, common);
3273
3274 iocb->ret = -ECANCELED;
3275
3276 if (iocb->aiocb) {
3277 blk_aio_cancel_async(iocb->aiocb);
3278 iocb->aiocb = NULL;
3279 }
3280 }
3281
3282 static const AIOCBInfo nvme_flush_aiocb_info = {
3283 .aiocb_size = sizeof(NvmeFlushAIOCB),
3284 .cancel_async = nvme_flush_cancel,
3285 .get_aio_context = nvme_get_aio_context,
3286 };
3287
3288 static void nvme_do_flush(NvmeFlushAIOCB *iocb);
3289
3290 static void nvme_flush_ns_cb(void *opaque, int ret)
3291 {
3292 NvmeFlushAIOCB *iocb = opaque;
3293 NvmeNamespace *ns = iocb->ns;
3294
3295 if (ret < 0) {
3296 iocb->ret = ret;
3297 goto out;
3298 } else if (iocb->ret < 0) {
3299 goto out;
3300 }
3301
3302 if (ns) {
3303 trace_pci_nvme_flush_ns(iocb->nsid);
3304
3305 iocb->ns = NULL;
3306 iocb->aiocb = blk_aio_flush(ns->blkconf.blk, nvme_flush_ns_cb, iocb);
3307 return;
3308 }
3309
3310 out:
3311 nvme_do_flush(iocb);
3312 }
3313
3314 static void nvme_do_flush(NvmeFlushAIOCB *iocb)
3315 {
3316 NvmeRequest *req = iocb->req;
3317 NvmeCtrl *n = nvme_ctrl(req);
3318 int i;
3319
3320 if (iocb->ret < 0) {
3321 goto done;
3322 }
3323
3324 if (iocb->broadcast) {
3325 for (i = iocb->nsid + 1; i <= NVME_MAX_NAMESPACES; i++) {
3326 iocb->ns = nvme_ns(n, i);
3327 if (iocb->ns) {
3328 iocb->nsid = i;
3329 break;
3330 }
3331 }
3332 }
3333
3334 if (!iocb->ns) {
3335 goto done;
3336 }
3337
3338 nvme_flush_ns_cb(iocb, 0);
3339 return;
3340
3341 done:
3342 iocb->common.cb(iocb->common.opaque, iocb->ret);
3343 qemu_aio_unref(iocb);
3344 }
3345
3346 static uint16_t nvme_flush(NvmeCtrl *n, NvmeRequest *req)
3347 {
3348 NvmeFlushAIOCB *iocb;
3349 uint32_t nsid = le32_to_cpu(req->cmd.nsid);
3350 uint16_t status;
3351
3352 iocb = qemu_aio_get(&nvme_flush_aiocb_info, NULL, nvme_misc_cb, req);
3353
3354 iocb->req = req;
3355 iocb->ret = 0;
3356 iocb->ns = NULL;
3357 iocb->nsid = 0;
3358 iocb->broadcast = (nsid == NVME_NSID_BROADCAST);
3359
3360 if (!iocb->broadcast) {
3361 if (!nvme_nsid_valid(n, nsid)) {
3362 status = NVME_INVALID_NSID | NVME_DNR;
3363 goto out;
3364 }
3365
3366 iocb->ns = nvme_ns(n, nsid);
3367 if (!iocb->ns) {
3368 status = NVME_INVALID_FIELD | NVME_DNR;
3369 goto out;
3370 }
3371
3372 iocb->nsid = nsid;
3373 }
3374
3375 req->aiocb = &iocb->common;
3376 nvme_do_flush(iocb);
3377
3378 return NVME_NO_COMPLETE;
3379
3380 out:
3381 qemu_aio_unref(iocb);
3382
3383 return status;
3384 }
3385
3386 static uint16_t nvme_read(NvmeCtrl *n, NvmeRequest *req)
3387 {
3388 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
3389 NvmeNamespace *ns = req->ns;
3390 uint64_t slba = le64_to_cpu(rw->slba);
3391 uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb) + 1;
3392 uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
3393 uint64_t data_size = nvme_l2b(ns, nlb);
3394 uint64_t mapped_size = data_size;
3395 uint64_t data_offset;
3396 BlockBackend *blk = ns->blkconf.blk;
3397 uint16_t status;
3398
3399 if (nvme_ns_ext(ns)) {
3400 mapped_size += nvme_m2b(ns, nlb);
3401
3402 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3403 bool pract = prinfo & NVME_PRINFO_PRACT;
3404
3405 if (pract && ns->lbaf.ms == nvme_pi_tuple_size(ns)) {
3406 mapped_size = data_size;
3407 }
3408 }
3409 }
3410
3411 trace_pci_nvme_read(nvme_cid(req), nvme_nsid(ns), nlb, mapped_size, slba);
3412
3413 status = nvme_check_mdts(n, mapped_size);
3414 if (status) {
3415 goto invalid;
3416 }
3417
3418 status = nvme_check_bounds(ns, slba, nlb);
3419 if (status) {
3420 goto invalid;
3421 }
3422
3423 if (ns->params.zoned) {
3424 status = nvme_check_zone_read(ns, slba, nlb);
3425 if (status) {
3426 trace_pci_nvme_err_zone_read_not_ok(slba, nlb, status);
3427 goto invalid;
3428 }
3429 }
3430
3431 if (NVME_ERR_REC_DULBE(ns->features.err_rec)) {
3432 status = nvme_check_dulbe(ns, slba, nlb);
3433 if (status) {
3434 goto invalid;
3435 }
3436 }
3437
3438 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3439 return nvme_dif_rw(n, req);
3440 }
3441
3442 status = nvme_map_data(n, nlb, req);
3443 if (status) {
3444 goto invalid;
3445 }
3446
3447 data_offset = nvme_l2b(ns, slba);
3448
3449 block_acct_start(blk_get_stats(blk), &req->acct, data_size,
3450 BLOCK_ACCT_READ);
3451 nvme_blk_read(blk, data_offset, BDRV_SECTOR_SIZE, nvme_rw_cb, req);
3452 return NVME_NO_COMPLETE;
3453
3454 invalid:
3455 block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_READ);
3456 return status | NVME_DNR;
3457 }
3458
3459 static void nvme_do_write_fdp(NvmeCtrl *n, NvmeRequest *req, uint64_t slba,
3460 uint32_t nlb)
3461 {
3462 NvmeNamespace *ns = req->ns;
3463 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
3464 uint64_t data_size = nvme_l2b(ns, nlb);
3465 uint32_t dw12 = le32_to_cpu(req->cmd.cdw12);
3466 uint8_t dtype = (dw12 >> 20) & 0xf;
3467 uint16_t pid = le16_to_cpu(rw->dspec);
3468 uint16_t ph, rg, ruhid;
3469 NvmeReclaimUnit *ru;
3470
3471 if (dtype != NVME_DIRECTIVE_DATA_PLACEMENT ||
3472 !nvme_parse_pid(ns, pid, &ph, &rg)) {
3473 ph = 0;
3474 rg = 0;
3475 }
3476
3477 ruhid = ns->fdp.phs[ph];
3478 ru = &ns->endgrp->fdp.ruhs[ruhid].rus[rg];
3479
3480 nvme_fdp_stat_inc(&ns->endgrp->fdp.hbmw, data_size);
3481 nvme_fdp_stat_inc(&ns->endgrp->fdp.mbmw, data_size);
3482
3483 while (nlb) {
3484 if (nlb < ru->ruamw) {
3485 ru->ruamw -= nlb;
3486 break;
3487 }
3488
3489 nlb -= ru->ruamw;
3490 nvme_update_ruh(n, ns, pid);
3491 }
3492 }
3493
3494 static uint16_t nvme_do_write(NvmeCtrl *n, NvmeRequest *req, bool append,
3495 bool wrz)
3496 {
3497 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
3498 NvmeNamespace *ns = req->ns;
3499 uint64_t slba = le64_to_cpu(rw->slba);
3500 uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb) + 1;
3501 uint16_t ctrl = le16_to_cpu(rw->control);
3502 uint8_t prinfo = NVME_RW_PRINFO(ctrl);
3503 uint64_t data_size = nvme_l2b(ns, nlb);
3504 uint64_t mapped_size = data_size;
3505 uint64_t data_offset;
3506 NvmeZone *zone;
3507 NvmeZonedResult *res = (NvmeZonedResult *)&req->cqe;
3508 BlockBackend *blk = ns->blkconf.blk;
3509 uint16_t status;
3510
3511 if (nvme_ns_ext(ns)) {
3512 mapped_size += nvme_m2b(ns, nlb);
3513
3514 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3515 bool pract = prinfo & NVME_PRINFO_PRACT;
3516
3517 if (pract && ns->lbaf.ms == nvme_pi_tuple_size(ns)) {
3518 mapped_size -= nvme_m2b(ns, nlb);
3519 }
3520 }
3521 }
3522
3523 trace_pci_nvme_write(nvme_cid(req), nvme_io_opc_str(rw->opcode),
3524 nvme_nsid(ns), nlb, mapped_size, slba);
3525
3526 if (!wrz) {
3527 status = nvme_check_mdts(n, mapped_size);
3528 if (status) {
3529 goto invalid;
3530 }
3531 }
3532
3533 status = nvme_check_bounds(ns, slba, nlb);
3534 if (status) {
3535 goto invalid;
3536 }
3537
3538 if (ns->params.zoned) {
3539 zone = nvme_get_zone_by_slba(ns, slba);
3540 assert(zone);
3541
3542 if (append) {
3543 bool piremap = !!(ctrl & NVME_RW_PIREMAP);
3544
3545 if (unlikely(zone->d.za & NVME_ZA_ZRWA_VALID)) {
3546 return NVME_INVALID_ZONE_OP | NVME_DNR;
3547 }
3548
3549 if (unlikely(slba != zone->d.zslba)) {
3550 trace_pci_nvme_err_append_not_at_start(slba, zone->d.zslba);
3551 status = NVME_INVALID_FIELD;
3552 goto invalid;
3553 }
3554
3555 if (n->params.zasl &&
3556 data_size > (uint64_t)n->page_size << n->params.zasl) {
3557 trace_pci_nvme_err_zasl(data_size);
3558 return NVME_INVALID_FIELD | NVME_DNR;
3559 }
3560
3561 slba = zone->w_ptr;
3562 rw->slba = cpu_to_le64(slba);
3563 res->slba = cpu_to_le64(slba);
3564
3565 switch (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3566 case NVME_ID_NS_DPS_TYPE_1:
3567 if (!piremap) {
3568 return NVME_INVALID_PROT_INFO | NVME_DNR;
3569 }
3570
3571 /* fallthrough */
3572
3573 case NVME_ID_NS_DPS_TYPE_2:
3574 if (piremap) {
3575 uint32_t reftag = le32_to_cpu(rw->reftag);
3576 rw->reftag = cpu_to_le32(reftag + (slba - zone->d.zslba));
3577 }
3578
3579 break;
3580
3581 case NVME_ID_NS_DPS_TYPE_3:
3582 if (piremap) {
3583 return NVME_INVALID_PROT_INFO | NVME_DNR;
3584 }
3585
3586 break;
3587 }
3588 }
3589
3590 status = nvme_check_zone_write(ns, zone, slba, nlb);
3591 if (status) {
3592 goto invalid;
3593 }
3594
3595 status = nvme_zrm_auto(n, ns, zone);
3596 if (status) {
3597 goto invalid;
3598 }
3599
3600 if (!(zone->d.za & NVME_ZA_ZRWA_VALID)) {
3601 zone->w_ptr += nlb;
3602 }
3603 } else if (ns->endgrp && ns->endgrp->fdp.enabled) {
3604 nvme_do_write_fdp(n, req, slba, nlb);
3605 }
3606
3607 data_offset = nvme_l2b(ns, slba);
3608
3609 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3610 return nvme_dif_rw(n, req);
3611 }
3612
3613 if (!wrz) {
3614 status = nvme_map_data(n, nlb, req);
3615 if (status) {
3616 goto invalid;
3617 }
3618
3619 block_acct_start(blk_get_stats(blk), &req->acct, data_size,
3620 BLOCK_ACCT_WRITE);
3621 nvme_blk_write(blk, data_offset, BDRV_SECTOR_SIZE, nvme_rw_cb, req);
3622 } else {
3623 req->aiocb = blk_aio_pwrite_zeroes(blk, data_offset, data_size,
3624 BDRV_REQ_MAY_UNMAP, nvme_rw_cb,
3625 req);
3626 }
3627
3628 return NVME_NO_COMPLETE;
3629
3630 invalid:
3631 block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_WRITE);
3632 return status | NVME_DNR;
3633 }
3634
3635 static inline uint16_t nvme_write(NvmeCtrl *n, NvmeRequest *req)
3636 {
3637 return nvme_do_write(n, req, false, false);
3638 }
3639
3640 static inline uint16_t nvme_write_zeroes(NvmeCtrl *n, NvmeRequest *req)
3641 {
3642 return nvme_do_write(n, req, false, true);
3643 }
3644
3645 static inline uint16_t nvme_zone_append(NvmeCtrl *n, NvmeRequest *req)
3646 {
3647 return nvme_do_write(n, req, true, false);
3648 }
3649
3650 static uint16_t nvme_get_mgmt_zone_slba_idx(NvmeNamespace *ns, NvmeCmd *c,
3651 uint64_t *slba, uint32_t *zone_idx)
3652 {
3653 uint32_t dw10 = le32_to_cpu(c->cdw10);
3654 uint32_t dw11 = le32_to_cpu(c->cdw11);
3655
3656 if (!ns->params.zoned) {
3657 trace_pci_nvme_err_invalid_opc(c->opcode);
3658 return NVME_INVALID_OPCODE | NVME_DNR;
3659 }
3660
3661 *slba = ((uint64_t)dw11) << 32 | dw10;
3662 if (unlikely(*slba >= ns->id_ns.nsze)) {
3663 trace_pci_nvme_err_invalid_lba_range(*slba, 0, ns->id_ns.nsze);
3664 *slba = 0;
3665 return NVME_LBA_RANGE | NVME_DNR;
3666 }
3667
3668 *zone_idx = nvme_zone_idx(ns, *slba);
3669 assert(*zone_idx < ns->num_zones);
3670
3671 return NVME_SUCCESS;
3672 }
3673
3674 typedef uint16_t (*op_handler_t)(NvmeNamespace *, NvmeZone *, NvmeZoneState,
3675 NvmeRequest *);
3676
3677 enum NvmeZoneProcessingMask {
3678 NVME_PROC_CURRENT_ZONE = 0,
3679 NVME_PROC_OPENED_ZONES = 1 << 0,
3680 NVME_PROC_CLOSED_ZONES = 1 << 1,
3681 NVME_PROC_READ_ONLY_ZONES = 1 << 2,
3682 NVME_PROC_FULL_ZONES = 1 << 3,
3683 };
3684
3685 static uint16_t nvme_open_zone(NvmeNamespace *ns, NvmeZone *zone,
3686 NvmeZoneState state, NvmeRequest *req)
3687 {
3688 NvmeZoneSendCmd *cmd = (NvmeZoneSendCmd *)&req->cmd;
3689 int flags = 0;
3690
3691 if (cmd->zsflags & NVME_ZSFLAG_ZRWA_ALLOC) {
3692 uint16_t ozcs = le16_to_cpu(ns->id_ns_zoned->ozcs);
3693
3694 if (!(ozcs & NVME_ID_NS_ZONED_OZCS_ZRWASUP)) {
3695 return NVME_INVALID_ZONE_OP | NVME_DNR;
3696 }
3697
3698 if (zone->w_ptr % ns->zns.zrwafg) {
3699 return NVME_NOZRWA | NVME_DNR;
3700 }
3701
3702 flags = NVME_ZRM_ZRWA;
3703 }
3704
3705 return nvme_zrm_open_flags(nvme_ctrl(req), ns, zone, flags);
3706 }
3707
3708 static uint16_t nvme_close_zone(NvmeNamespace *ns, NvmeZone *zone,
3709 NvmeZoneState state, NvmeRequest *req)
3710 {
3711 return nvme_zrm_close(ns, zone);
3712 }
3713
3714 static uint16_t nvme_finish_zone(NvmeNamespace *ns, NvmeZone *zone,
3715 NvmeZoneState state, NvmeRequest *req)
3716 {
3717 return nvme_zrm_finish(ns, zone);
3718 }
3719
3720 static uint16_t nvme_offline_zone(NvmeNamespace *ns, NvmeZone *zone,
3721 NvmeZoneState state, NvmeRequest *req)
3722 {
3723 switch (state) {
3724 case NVME_ZONE_STATE_READ_ONLY:
3725 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_OFFLINE);
3726 /* fall through */
3727 case NVME_ZONE_STATE_OFFLINE:
3728 return NVME_SUCCESS;
3729 default:
3730 return NVME_ZONE_INVAL_TRANSITION;
3731 }
3732 }
3733
3734 static uint16_t nvme_set_zd_ext(NvmeNamespace *ns, NvmeZone *zone)
3735 {
3736 uint16_t status;
3737 uint8_t state = nvme_get_zone_state(zone);
3738
3739 if (state == NVME_ZONE_STATE_EMPTY) {
3740 status = nvme_aor_check(ns, 1, 0);
3741 if (status) {
3742 return status;
3743 }
3744 nvme_aor_inc_active(ns);
3745 zone->d.za |= NVME_ZA_ZD_EXT_VALID;
3746 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_CLOSED);
3747 return NVME_SUCCESS;
3748 }
3749
3750 return NVME_ZONE_INVAL_TRANSITION;
3751 }
3752
3753 static uint16_t nvme_bulk_proc_zone(NvmeNamespace *ns, NvmeZone *zone,
3754 enum NvmeZoneProcessingMask proc_mask,
3755 op_handler_t op_hndlr, NvmeRequest *req)
3756 {
3757 uint16_t status = NVME_SUCCESS;
3758 NvmeZoneState zs = nvme_get_zone_state(zone);
3759 bool proc_zone;
3760
3761 switch (zs) {
3762 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
3763 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
3764 proc_zone = proc_mask & NVME_PROC_OPENED_ZONES;
3765 break;
3766 case NVME_ZONE_STATE_CLOSED:
3767 proc_zone = proc_mask & NVME_PROC_CLOSED_ZONES;
3768 break;
3769 case NVME_ZONE_STATE_READ_ONLY:
3770 proc_zone = proc_mask & NVME_PROC_READ_ONLY_ZONES;
3771 break;
3772 case NVME_ZONE_STATE_FULL:
3773 proc_zone = proc_mask & NVME_PROC_FULL_ZONES;
3774 break;
3775 default:
3776 proc_zone = false;
3777 }
3778
3779 if (proc_zone) {
3780 status = op_hndlr(ns, zone, zs, req);
3781 }
3782
3783 return status;
3784 }
3785
3786 static uint16_t nvme_do_zone_op(NvmeNamespace *ns, NvmeZone *zone,
3787 enum NvmeZoneProcessingMask proc_mask,
3788 op_handler_t op_hndlr, NvmeRequest *req)
3789 {
3790 NvmeZone *next;
3791 uint16_t status = NVME_SUCCESS;
3792 int i;
3793
3794 if (!proc_mask) {
3795 status = op_hndlr(ns, zone, nvme_get_zone_state(zone), req);
3796 } else {
3797 if (proc_mask & NVME_PROC_CLOSED_ZONES) {
3798 QTAILQ_FOREACH_SAFE(zone, &ns->closed_zones, entry, next) {
3799 status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
3800 req);
3801 if (status && status != NVME_NO_COMPLETE) {
3802 goto out;
3803 }
3804 }
3805 }
3806 if (proc_mask & NVME_PROC_OPENED_ZONES) {
3807 QTAILQ_FOREACH_SAFE(zone, &ns->imp_open_zones, entry, next) {
3808 status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
3809 req);
3810 if (status && status != NVME_NO_COMPLETE) {
3811 goto out;
3812 }
3813 }
3814
3815 QTAILQ_FOREACH_SAFE(zone, &ns->exp_open_zones, entry, next) {
3816 status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
3817 req);
3818 if (status && status != NVME_NO_COMPLETE) {
3819 goto out;
3820 }
3821 }
3822 }
3823 if (proc_mask & NVME_PROC_FULL_ZONES) {
3824 QTAILQ_FOREACH_SAFE(zone, &ns->full_zones, entry, next) {
3825 status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
3826 req);
3827 if (status && status != NVME_NO_COMPLETE) {
3828 goto out;
3829 }
3830 }
3831 }
3832
3833 if (proc_mask & NVME_PROC_READ_ONLY_ZONES) {
3834 for (i = 0; i < ns->num_zones; i++, zone++) {
3835 status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
3836 req);
3837 if (status && status != NVME_NO_COMPLETE) {
3838 goto out;
3839 }
3840 }
3841 }
3842 }
3843
3844 out:
3845 return status;
3846 }
3847
3848 typedef struct NvmeZoneResetAIOCB {
3849 BlockAIOCB common;
3850 BlockAIOCB *aiocb;
3851 NvmeRequest *req;
3852 int ret;
3853
3854 bool all;
3855 int idx;
3856 NvmeZone *zone;
3857 } NvmeZoneResetAIOCB;
3858
3859 static void nvme_zone_reset_cancel(BlockAIOCB *aiocb)
3860 {
3861 NvmeZoneResetAIOCB *iocb = container_of(aiocb, NvmeZoneResetAIOCB, common);
3862 NvmeRequest *req = iocb->req;
3863 NvmeNamespace *ns = req->ns;
3864
3865 iocb->idx = ns->num_zones;
3866
3867 iocb->ret = -ECANCELED;
3868
3869 if (iocb->aiocb) {
3870 blk_aio_cancel_async(iocb->aiocb);
3871 iocb->aiocb = NULL;
3872 }
3873 }
3874
3875 static const AIOCBInfo nvme_zone_reset_aiocb_info = {
3876 .aiocb_size = sizeof(NvmeZoneResetAIOCB),
3877 .cancel_async = nvme_zone_reset_cancel,
3878 };
3879
3880 static void nvme_zone_reset_cb(void *opaque, int ret);
3881
3882 static void nvme_zone_reset_epilogue_cb(void *opaque, int ret)
3883 {
3884 NvmeZoneResetAIOCB *iocb = opaque;
3885 NvmeRequest *req = iocb->req;
3886 NvmeNamespace *ns = req->ns;
3887 int64_t moff;
3888 int count;
3889
3890 if (ret < 0 || iocb->ret < 0 || !ns->lbaf.ms) {
3891 goto out;
3892 }
3893
3894 moff = nvme_moff(ns, iocb->zone->d.zslba);
3895 count = nvme_m2b(ns, ns->zone_size);
3896
3897 iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk, moff, count,
3898 BDRV_REQ_MAY_UNMAP,
3899 nvme_zone_reset_cb, iocb);
3900 return;
3901
3902 out:
3903 nvme_zone_reset_cb(iocb, ret);
3904 }
3905
3906 static void nvme_zone_reset_cb(void *opaque, int ret)
3907 {
3908 NvmeZoneResetAIOCB *iocb = opaque;
3909 NvmeRequest *req = iocb->req;
3910 NvmeNamespace *ns = req->ns;
3911
3912 if (iocb->ret < 0) {
3913 goto done;
3914 } else if (ret < 0) {
3915 iocb->ret = ret;
3916 goto done;
3917 }
3918
3919 if (iocb->zone) {
3920 nvme_zrm_reset(ns, iocb->zone);
3921
3922 if (!iocb->all) {
3923 goto done;
3924 }
3925 }
3926
3927 while (iocb->idx < ns->num_zones) {
3928 NvmeZone *zone = &ns->zone_array[iocb->idx++];
3929
3930 switch (nvme_get_zone_state(zone)) {
3931 case NVME_ZONE_STATE_EMPTY:
3932 if (!iocb->all) {
3933 goto done;
3934 }
3935
3936 continue;
3937
3938 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
3939 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
3940 case NVME_ZONE_STATE_CLOSED:
3941 case NVME_ZONE_STATE_FULL:
3942 iocb->zone = zone;
3943 break;
3944
3945 default:
3946 continue;
3947 }
3948
3949 trace_pci_nvme_zns_zone_reset(zone->d.zslba);
3950
3951 iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk,
3952 nvme_l2b(ns, zone->d.zslba),
3953 nvme_l2b(ns, ns->zone_size),
3954 BDRV_REQ_MAY_UNMAP,
3955 nvme_zone_reset_epilogue_cb,
3956 iocb);
3957 return;
3958 }
3959
3960 done:
3961 iocb->aiocb = NULL;
3962
3963 iocb->common.cb(iocb->common.opaque, iocb->ret);
3964 qemu_aio_unref(iocb);
3965 }
3966
3967 static uint16_t nvme_zone_mgmt_send_zrwa_flush(NvmeCtrl *n, NvmeZone *zone,
3968 uint64_t elba, NvmeRequest *req)
3969 {
3970 NvmeNamespace *ns = req->ns;
3971 uint16_t ozcs = le16_to_cpu(ns->id_ns_zoned->ozcs);
3972 uint64_t wp = zone->d.wp;
3973 uint32_t nlb = elba - wp + 1;
3974 uint16_t status;
3975
3976
3977 if (!(ozcs & NVME_ID_NS_ZONED_OZCS_ZRWASUP)) {
3978 return NVME_INVALID_ZONE_OP | NVME_DNR;
3979 }
3980
3981 if (!(zone->d.za & NVME_ZA_ZRWA_VALID)) {
3982 return NVME_INVALID_FIELD | NVME_DNR;
3983 }
3984
3985 if (elba < wp || elba > wp + ns->zns.zrwas) {
3986 return NVME_ZONE_BOUNDARY_ERROR | NVME_DNR;
3987 }
3988
3989 if (nlb % ns->zns.zrwafg) {
3990 return NVME_INVALID_FIELD | NVME_DNR;
3991 }
3992
3993 status = nvme_zrm_auto(n, ns, zone);
3994 if (status) {
3995 return status;
3996 }
3997
3998 zone->w_ptr += nlb;
3999
4000 nvme_advance_zone_wp(ns, zone, nlb);
4001
4002 return NVME_SUCCESS;
4003 }
4004
4005 static uint16_t nvme_zone_mgmt_send(NvmeCtrl *n, NvmeRequest *req)
4006 {
4007 NvmeZoneSendCmd *cmd = (NvmeZoneSendCmd *)&req->cmd;
4008 NvmeNamespace *ns = req->ns;
4009 NvmeZone *zone;
4010 NvmeZoneResetAIOCB *iocb;
4011 uint8_t *zd_ext;
4012 uint64_t slba = 0;
4013 uint32_t zone_idx = 0;
4014 uint16_t status;
4015 uint8_t action = cmd->zsa;
4016 bool all;
4017 enum NvmeZoneProcessingMask proc_mask = NVME_PROC_CURRENT_ZONE;
4018
4019 all = cmd->zsflags & NVME_ZSFLAG_SELECT_ALL;
4020
4021 req->status = NVME_SUCCESS;
4022
4023 if (!all) {
4024 status = nvme_get_mgmt_zone_slba_idx(ns, &req->cmd, &slba, &zone_idx);
4025 if (status) {
4026 return status;
4027 }
4028 }
4029
4030 zone = &ns->zone_array[zone_idx];
4031 if (slba != zone->d.zslba && action != NVME_ZONE_ACTION_ZRWA_FLUSH) {
4032 trace_pci_nvme_err_unaligned_zone_cmd(action, slba, zone->d.zslba);
4033 return NVME_INVALID_FIELD | NVME_DNR;
4034 }
4035
4036 switch (action) {
4037
4038 case NVME_ZONE_ACTION_OPEN:
4039 if (all) {
4040 proc_mask = NVME_PROC_CLOSED_ZONES;
4041 }
4042 trace_pci_nvme_open_zone(slba, zone_idx, all);
4043 status = nvme_do_zone_op(ns, zone, proc_mask, nvme_open_zone, req);
4044 break;
4045
4046 case NVME_ZONE_ACTION_CLOSE:
4047 if (all) {
4048 proc_mask = NVME_PROC_OPENED_ZONES;
4049 }
4050 trace_pci_nvme_close_zone(slba, zone_idx, all);
4051 status = nvme_do_zone_op(ns, zone, proc_mask, nvme_close_zone, req);
4052 break;
4053
4054 case NVME_ZONE_ACTION_FINISH:
4055 if (all) {
4056 proc_mask = NVME_PROC_OPENED_ZONES | NVME_PROC_CLOSED_ZONES;
4057 }
4058 trace_pci_nvme_finish_zone(slba, zone_idx, all);
4059 status = nvme_do_zone_op(ns, zone, proc_mask, nvme_finish_zone, req);
4060 break;
4061
4062 case NVME_ZONE_ACTION_RESET:
4063 trace_pci_nvme_reset_zone(slba, zone_idx, all);
4064
4065 iocb = blk_aio_get(&nvme_zone_reset_aiocb_info, ns->blkconf.blk,
4066 nvme_misc_cb, req);
4067
4068 iocb->req = req;
4069 iocb->ret = 0;
4070 iocb->all = all;
4071 iocb->idx = zone_idx;
4072 iocb->zone = NULL;
4073
4074 req->aiocb = &iocb->common;
4075 nvme_zone_reset_cb(iocb, 0);
4076
4077 return NVME_NO_COMPLETE;
4078
4079 case NVME_ZONE_ACTION_OFFLINE:
4080 if (all) {
4081 proc_mask = NVME_PROC_READ_ONLY_ZONES;
4082 }
4083 trace_pci_nvme_offline_zone(slba, zone_idx, all);
4084 status = nvme_do_zone_op(ns, zone, proc_mask, nvme_offline_zone, req);
4085 break;
4086
4087 case NVME_ZONE_ACTION_SET_ZD_EXT:
4088 trace_pci_nvme_set_descriptor_extension(slba, zone_idx);
4089 if (all || !ns->params.zd_extension_size) {
4090 return NVME_INVALID_FIELD | NVME_DNR;
4091 }
4092 zd_ext = nvme_get_zd_extension(ns, zone_idx);
4093 status = nvme_h2c(n, zd_ext, ns->params.zd_extension_size, req);
4094 if (status) {
4095 trace_pci_nvme_err_zd_extension_map_error(zone_idx);
4096 return status;
4097 }
4098
4099 status = nvme_set_zd_ext(ns, zone);
4100 if (status == NVME_SUCCESS) {
4101 trace_pci_nvme_zd_extension_set(zone_idx);
4102 return status;
4103 }
4104 break;
4105
4106 case NVME_ZONE_ACTION_ZRWA_FLUSH:
4107 if (all) {
4108 return NVME_INVALID_FIELD | NVME_DNR;
4109 }
4110
4111 return nvme_zone_mgmt_send_zrwa_flush(n, zone, slba, req);
4112
4113 default:
4114 trace_pci_nvme_err_invalid_mgmt_action(action);
4115 status = NVME_INVALID_FIELD;
4116 }
4117
4118 if (status == NVME_ZONE_INVAL_TRANSITION) {
4119 trace_pci_nvme_err_invalid_zone_state_transition(action, slba,
4120 zone->d.za);
4121 }
4122 if (status) {
4123 status |= NVME_DNR;
4124 }
4125
4126 return status;
4127 }
4128
4129 static bool nvme_zone_matches_filter(uint32_t zafs, NvmeZone *zl)
4130 {
4131 NvmeZoneState zs = nvme_get_zone_state(zl);
4132
4133 switch (zafs) {
4134 case NVME_ZONE_REPORT_ALL:
4135 return true;
4136 case NVME_ZONE_REPORT_EMPTY:
4137 return zs == NVME_ZONE_STATE_EMPTY;
4138 case NVME_ZONE_REPORT_IMPLICITLY_OPEN:
4139 return zs == NVME_ZONE_STATE_IMPLICITLY_OPEN;
4140 case NVME_ZONE_REPORT_EXPLICITLY_OPEN:
4141 return zs == NVME_ZONE_STATE_EXPLICITLY_OPEN;
4142 case NVME_ZONE_REPORT_CLOSED:
4143 return zs == NVME_ZONE_STATE_CLOSED;
4144 case NVME_ZONE_REPORT_FULL:
4145 return zs == NVME_ZONE_STATE_FULL;
4146 case NVME_ZONE_REPORT_READ_ONLY:
4147 return zs == NVME_ZONE_STATE_READ_ONLY;
4148 case NVME_ZONE_REPORT_OFFLINE:
4149 return zs == NVME_ZONE_STATE_OFFLINE;
4150 default:
4151 return false;
4152 }
4153 }
4154
4155 static uint16_t nvme_zone_mgmt_recv(NvmeCtrl *n, NvmeRequest *req)
4156 {
4157 NvmeCmd *cmd = (NvmeCmd *)&req->cmd;
4158 NvmeNamespace *ns = req->ns;
4159 /* cdw12 is zero-based number of dwords to return. Convert to bytes */
4160 uint32_t data_size = (le32_to_cpu(cmd->cdw12) + 1) << 2;
4161 uint32_t dw13 = le32_to_cpu(cmd->cdw13);
4162 uint32_t zone_idx, zra, zrasf, partial;
4163 uint64_t max_zones, nr_zones = 0;
4164 uint16_t status;
4165 uint64_t slba;
4166 NvmeZoneDescr *z;
4167 NvmeZone *zone;
4168 NvmeZoneReportHeader *header;
4169 void *buf, *buf_p;
4170 size_t zone_entry_sz;
4171 int i;
4172
4173 req->status = NVME_SUCCESS;
4174
4175 status = nvme_get_mgmt_zone_slba_idx(ns, cmd, &slba, &zone_idx);
4176 if (status) {
4177 return status;
4178 }
4179
4180 zra = dw13 & 0xff;
4181 if (zra != NVME_ZONE_REPORT && zra != NVME_ZONE_REPORT_EXTENDED) {
4182 return NVME_INVALID_FIELD | NVME_DNR;
4183 }
4184 if (zra == NVME_ZONE_REPORT_EXTENDED && !ns->params.zd_extension_size) {
4185 return NVME_INVALID_FIELD | NVME_DNR;
4186 }
4187
4188 zrasf = (dw13 >> 8) & 0xff;
4189 if (zrasf > NVME_ZONE_REPORT_OFFLINE) {
4190 return NVME_INVALID_FIELD | NVME_DNR;
4191 }
4192
4193 if (data_size < sizeof(NvmeZoneReportHeader)) {
4194 return NVME_INVALID_FIELD | NVME_DNR;
4195 }
4196
4197 status = nvme_check_mdts(n, data_size);
4198 if (status) {
4199 return status;
4200 }
4201
4202 partial = (dw13 >> 16) & 0x01;
4203
4204 zone_entry_sz = sizeof(NvmeZoneDescr);
4205 if (zra == NVME_ZONE_REPORT_EXTENDED) {
4206 zone_entry_sz += ns->params.zd_extension_size;
4207 }
4208
4209 max_zones = (data_size - sizeof(NvmeZoneReportHeader)) / zone_entry_sz;
4210 buf = g_malloc0(data_size);
4211
4212 zone = &ns->zone_array[zone_idx];
4213 for (i = zone_idx; i < ns->num_zones; i++) {
4214 if (partial && nr_zones >= max_zones) {
4215 break;
4216 }
4217 if (nvme_zone_matches_filter(zrasf, zone++)) {
4218 nr_zones++;
4219 }
4220 }
4221 header = buf;
4222 header->nr_zones = cpu_to_le64(nr_zones);
4223
4224 buf_p = buf + sizeof(NvmeZoneReportHeader);
4225 for (; zone_idx < ns->num_zones && max_zones > 0; zone_idx++) {
4226 zone = &ns->zone_array[zone_idx];
4227 if (nvme_zone_matches_filter(zrasf, zone)) {
4228 z = buf_p;
4229 buf_p += sizeof(NvmeZoneDescr);
4230
4231 z->zt = zone->d.zt;
4232 z->zs = zone->d.zs;
4233 z->zcap = cpu_to_le64(zone->d.zcap);
4234 z->zslba = cpu_to_le64(zone->d.zslba);
4235 z->za = zone->d.za;
4236
4237 if (nvme_wp_is_valid(zone)) {
4238 z->wp = cpu_to_le64(zone->d.wp);
4239 } else {
4240 z->wp = cpu_to_le64(~0ULL);
4241 }
4242
4243 if (zra == NVME_ZONE_REPORT_EXTENDED) {
4244 if (zone->d.za & NVME_ZA_ZD_EXT_VALID) {
4245 memcpy(buf_p, nvme_get_zd_extension(ns, zone_idx),
4246 ns->params.zd_extension_size);
4247 }
4248 buf_p += ns->params.zd_extension_size;
4249 }
4250
4251 max_zones--;
4252 }
4253 }
4254
4255 status = nvme_c2h(n, (uint8_t *)buf, data_size, req);
4256
4257 g_free(buf);
4258
4259 return status;
4260 }
4261
4262 static uint16_t nvme_io_mgmt_recv_ruhs(NvmeCtrl *n, NvmeRequest *req,
4263 size_t len)
4264 {
4265 NvmeNamespace *ns = req->ns;
4266 NvmeEnduranceGroup *endgrp;
4267 NvmeRuhStatus *hdr;
4268 NvmeRuhStatusDescr *ruhsd;
4269 unsigned int nruhsd;
4270 uint16_t rg, ph, *ruhid;
4271 size_t trans_len;
4272 g_autofree uint8_t *buf = NULL;
4273
4274 if (!n->subsys) {
4275 return NVME_INVALID_FIELD | NVME_DNR;
4276 }
4277
4278 if (ns->params.nsid == 0 || ns->params.nsid == 0xffffffff) {
4279 return NVME_INVALID_NSID | NVME_DNR;
4280 }
4281
4282 if (!n->subsys->endgrp.fdp.enabled) {
4283 return NVME_FDP_DISABLED | NVME_DNR;
4284 }
4285
4286 endgrp = ns->endgrp;
4287
4288 nruhsd = ns->fdp.nphs * endgrp->fdp.nrg;
4289 trans_len = sizeof(NvmeRuhStatus) + nruhsd * sizeof(NvmeRuhStatusDescr);
4290 buf = g_malloc(trans_len);
4291
4292 trans_len = MIN(trans_len, len);
4293
4294 hdr = (NvmeRuhStatus *)buf;
4295 ruhsd = (NvmeRuhStatusDescr *)(buf + sizeof(NvmeRuhStatus));
4296
4297 hdr->nruhsd = cpu_to_le16(nruhsd);
4298
4299 ruhid = ns->fdp.phs;
4300
4301 for (ph = 0; ph < ns->fdp.nphs; ph++, ruhid++) {
4302 NvmeRuHandle *ruh = &endgrp->fdp.ruhs[*ruhid];
4303
4304 for (rg = 0; rg < endgrp->fdp.nrg; rg++, ruhsd++) {
4305 uint16_t pid = nvme_make_pid(ns, rg, ph);
4306
4307 ruhsd->pid = cpu_to_le16(pid);
4308 ruhsd->ruhid = *ruhid;
4309 ruhsd->earutr = 0;
4310 ruhsd->ruamw = cpu_to_le64(ruh->rus[rg].ruamw);
4311 }
4312 }
4313
4314 return nvme_c2h(n, buf, trans_len, req);
4315 }
4316
4317 static uint16_t nvme_io_mgmt_recv(NvmeCtrl *n, NvmeRequest *req)
4318 {
4319 NvmeCmd *cmd = &req->cmd;
4320 uint32_t cdw10 = le32_to_cpu(cmd->cdw10);
4321 uint32_t numd = le32_to_cpu(cmd->cdw11);
4322 uint8_t mo = (cdw10 & 0xff);
4323 size_t len = (numd + 1) << 2;
4324
4325 switch (mo) {
4326 case NVME_IOMR_MO_NOP:
4327 return 0;
4328 case NVME_IOMR_MO_RUH_STATUS:
4329 return nvme_io_mgmt_recv_ruhs(n, req, len);
4330 default:
4331 return NVME_INVALID_FIELD | NVME_DNR;
4332 };
4333 }
4334
4335 static uint16_t nvme_io_mgmt_send_ruh_update(NvmeCtrl *n, NvmeRequest *req)
4336 {
4337 NvmeCmd *cmd = &req->cmd;
4338 NvmeNamespace *ns = req->ns;
4339 uint32_t cdw10 = le32_to_cpu(cmd->cdw10);
4340 uint16_t ret = NVME_SUCCESS;
4341 uint32_t npid = (cdw10 >> 1) + 1;
4342 unsigned int i = 0;
4343 g_autofree uint16_t *pids = NULL;
4344 uint32_t maxnpid = n->subsys->endgrp.fdp.nrg * n->subsys->endgrp.fdp.nruh;
4345
4346 if (unlikely(npid >= MIN(NVME_FDP_MAXPIDS, maxnpid))) {
4347 return NVME_INVALID_FIELD | NVME_DNR;
4348 }
4349
4350 pids = g_new(uint16_t, npid);
4351
4352 ret = nvme_h2c(n, pids, npid * sizeof(uint16_t), req);
4353 if (ret) {
4354 return ret;
4355 }
4356
4357 for (; i < npid; i++) {
4358 if (!nvme_update_ruh(n, ns, pids[i])) {
4359 return NVME_INVALID_FIELD | NVME_DNR;
4360 }
4361 }
4362
4363 return ret;
4364 }
4365
4366 static uint16_t nvme_io_mgmt_send(NvmeCtrl *n, NvmeRequest *req)
4367 {
4368 NvmeCmd *cmd = &req->cmd;
4369 uint32_t cdw10 = le32_to_cpu(cmd->cdw10);
4370 uint8_t mo = (cdw10 & 0xff);
4371
4372 switch (mo) {
4373 case NVME_IOMS_MO_NOP:
4374 return 0;
4375 case NVME_IOMS_MO_RUH_UPDATE:
4376 return nvme_io_mgmt_send_ruh_update(n, req);
4377 default:
4378 return NVME_INVALID_FIELD | NVME_DNR;
4379 };
4380 }
4381
4382 static uint16_t nvme_io_cmd(NvmeCtrl *n, NvmeRequest *req)
4383 {
4384 NvmeNamespace *ns;
4385 uint32_t nsid = le32_to_cpu(req->cmd.nsid);
4386
4387 trace_pci_nvme_io_cmd(nvme_cid(req), nsid, nvme_sqid(req),
4388 req->cmd.opcode, nvme_io_opc_str(req->cmd.opcode));
4389
4390 if (!nvme_nsid_valid(n, nsid)) {
4391 return NVME_INVALID_NSID | NVME_DNR;
4392 }
4393
4394 /*
4395 * In the base NVM command set, Flush may apply to all namespaces
4396 * (indicated by NSID being set to FFFFFFFFh). But if that feature is used
4397 * along with TP 4056 (Namespace Types), it may be pretty screwed up.
4398 *
4399 * If NSID is indeed set to FFFFFFFFh, we simply cannot associate the
4400 * opcode with a specific command since we cannot determine a unique I/O
4401 * command set. Opcode 0h could have any other meaning than something
4402 * equivalent to flushing and say it DOES have completely different
4403 * semantics in some other command set - does an NSID of FFFFFFFFh then
4404 * mean "for all namespaces, apply whatever command set specific command
4405 * that uses the 0h opcode?" Or does it mean "for all namespaces, apply
4406 * whatever command that uses the 0h opcode if, and only if, it allows NSID
4407 * to be FFFFFFFFh"?
4408 *
4409 * Anyway (and luckily), for now, we do not care about this since the
4410 * device only supports namespace types that includes the NVM Flush command
4411 * (NVM and Zoned), so always do an NVM Flush.
4412 */
4413 if (req->cmd.opcode == NVME_CMD_FLUSH) {
4414 return nvme_flush(n, req);
4415 }
4416
4417 ns = nvme_ns(n, nsid);
4418 if (unlikely(!ns)) {
4419 return NVME_INVALID_FIELD | NVME_DNR;
4420 }
4421
4422 if (!(ns->iocs[req->cmd.opcode] & NVME_CMD_EFF_CSUPP)) {
4423 trace_pci_nvme_err_invalid_opc(req->cmd.opcode);
4424 return NVME_INVALID_OPCODE | NVME_DNR;
4425 }
4426
4427 if (ns->status) {
4428 return ns->status;
4429 }
4430
4431 if (NVME_CMD_FLAGS_FUSE(req->cmd.flags)) {
4432 return NVME_INVALID_FIELD;
4433 }
4434
4435 req->ns = ns;
4436
4437 switch (req->cmd.opcode) {
4438 case NVME_CMD_WRITE_ZEROES:
4439 return nvme_write_zeroes(n, req);
4440 case NVME_CMD_ZONE_APPEND:
4441 return nvme_zone_append(n, req);
4442 case NVME_CMD_WRITE:
4443 return nvme_write(n, req);
4444 case NVME_CMD_READ:
4445 return nvme_read(n, req);
4446 case NVME_CMD_COMPARE:
4447 return nvme_compare(n, req);
4448 case NVME_CMD_DSM:
4449 return nvme_dsm(n, req);
4450 case NVME_CMD_VERIFY:
4451 return nvme_verify(n, req);
4452 case NVME_CMD_COPY:
4453 return nvme_copy(n, req);
4454 case NVME_CMD_ZONE_MGMT_SEND:
4455 return nvme_zone_mgmt_send(n, req);
4456 case NVME_CMD_ZONE_MGMT_RECV:
4457 return nvme_zone_mgmt_recv(n, req);
4458 case NVME_CMD_IO_MGMT_RECV:
4459 return nvme_io_mgmt_recv(n, req);
4460 case NVME_CMD_IO_MGMT_SEND:
4461 return nvme_io_mgmt_send(n, req);
4462 default:
4463 assert(false);
4464 }
4465
4466 return NVME_INVALID_OPCODE | NVME_DNR;
4467 }
4468
4469 static void nvme_cq_notifier(EventNotifier *e)
4470 {
4471 NvmeCQueue *cq = container_of(e, NvmeCQueue, notifier);
4472 NvmeCtrl *n = cq->ctrl;
4473
4474 if (!event_notifier_test_and_clear(e)) {
4475 return;
4476 }
4477
4478 nvme_update_cq_head(cq);
4479
4480 if (cq->tail == cq->head) {
4481 if (cq->irq_enabled) {
4482 n->cq_pending--;
4483 }
4484
4485 nvme_irq_deassert(n, cq);
4486 }
4487
4488 qemu_bh_schedule(cq->bh);
4489 }
4490
4491 static int nvme_init_cq_ioeventfd(NvmeCQueue *cq)
4492 {
4493 NvmeCtrl *n = cq->ctrl;
4494 uint16_t offset = (cq->cqid << 3) + (1 << 2);
4495 int ret;
4496
4497 ret = event_notifier_init(&cq->notifier, 0);
4498 if (ret < 0) {
4499 return ret;
4500 }
4501
4502 event_notifier_set_handler(&cq->notifier, nvme_cq_notifier);
4503 memory_region_add_eventfd(&n->iomem,
4504 0x1000 + offset, 4, false, 0, &cq->notifier);
4505
4506 return 0;
4507 }
4508
4509 static void nvme_sq_notifier(EventNotifier *e)
4510 {
4511 NvmeSQueue *sq = container_of(e, NvmeSQueue, notifier);
4512
4513 if (!event_notifier_test_and_clear(e)) {
4514 return;
4515 }
4516
4517 nvme_process_sq(sq);
4518 }
4519
4520 static int nvme_init_sq_ioeventfd(NvmeSQueue *sq)
4521 {
4522 NvmeCtrl *n = sq->ctrl;
4523 uint16_t offset = sq->sqid << 3;
4524 int ret;
4525
4526 ret = event_notifier_init(&sq->notifier, 0);
4527 if (ret < 0) {
4528 return ret;
4529 }
4530
4531 event_notifier_set_handler(&sq->notifier, nvme_sq_notifier);
4532 memory_region_add_eventfd(&n->iomem,
4533 0x1000 + offset, 4, false, 0, &sq->notifier);
4534
4535 return 0;
4536 }
4537
4538 static void nvme_free_sq(NvmeSQueue *sq, NvmeCtrl *n)
4539 {
4540 uint16_t offset = sq->sqid << 3;
4541
4542 n->sq[sq->sqid] = NULL;
4543 qemu_bh_delete(sq->bh);
4544 if (sq->ioeventfd_enabled) {
4545 memory_region_del_eventfd(&n->iomem,
4546 0x1000 + offset, 4, false, 0, &sq->notifier);
4547 event_notifier_set_handler(&sq->notifier, NULL);
4548 event_notifier_cleanup(&sq->notifier);
4549 }
4550 g_free(sq->io_req);
4551 if (sq->sqid) {
4552 g_free(sq);
4553 }
4554 }
4555
4556 static uint16_t nvme_del_sq(NvmeCtrl *n, NvmeRequest *req)
4557 {
4558 NvmeDeleteQ *c = (NvmeDeleteQ *)&req->cmd;
4559 NvmeRequest *r, *next;
4560 NvmeSQueue *sq;
4561 NvmeCQueue *cq;
4562 uint16_t qid = le16_to_cpu(c->qid);
4563
4564 if (unlikely(!qid || nvme_check_sqid(n, qid))) {
4565 trace_pci_nvme_err_invalid_del_sq(qid);
4566 return NVME_INVALID_QID | NVME_DNR;
4567 }
4568
4569 trace_pci_nvme_del_sq(qid);
4570
4571 sq = n->sq[qid];
4572 while (!QTAILQ_EMPTY(&sq->out_req_list)) {
4573 r = QTAILQ_FIRST(&sq->out_req_list);
4574 assert(r->aiocb);
4575 blk_aio_cancel(r->aiocb);
4576 }
4577
4578 assert(QTAILQ_EMPTY(&sq->out_req_list));
4579
4580 if (!nvme_check_cqid(n, sq->cqid)) {
4581 cq = n->cq[sq->cqid];
4582 QTAILQ_REMOVE(&cq->sq_list, sq, entry);
4583
4584 nvme_post_cqes(cq);
4585 QTAILQ_FOREACH_SAFE(r, &cq->req_list, entry, next) {
4586 if (r->sq == sq) {
4587 QTAILQ_REMOVE(&cq->req_list, r, entry);
4588 QTAILQ_INSERT_TAIL(&sq->req_list, r, entry);
4589 }
4590 }
4591 }
4592
4593 nvme_free_sq(sq, n);
4594 return NVME_SUCCESS;
4595 }
4596
4597 static void nvme_init_sq(NvmeSQueue *sq, NvmeCtrl *n, uint64_t dma_addr,
4598 uint16_t sqid, uint16_t cqid, uint16_t size)
4599 {
4600 int i;
4601 NvmeCQueue *cq;
4602
4603 sq->ctrl = n;
4604 sq->dma_addr = dma_addr;
4605 sq->sqid = sqid;
4606 sq->size = size;
4607 sq->cqid = cqid;
4608 sq->head = sq->tail = 0;
4609 sq->io_req = g_new0(NvmeRequest, sq->size);
4610
4611 QTAILQ_INIT(&sq->req_list);
4612 QTAILQ_INIT(&sq->out_req_list);
4613 for (i = 0; i < sq->size; i++) {
4614 sq->io_req[i].sq = sq;
4615 QTAILQ_INSERT_TAIL(&(sq->req_list), &sq->io_req[i], entry);
4616 }
4617
4618 sq->bh = qemu_bh_new_guarded(nvme_process_sq, sq,
4619 &DEVICE(sq->ctrl)->mem_reentrancy_guard);
4620
4621 if (n->dbbuf_enabled) {
4622 sq->db_addr = n->dbbuf_dbs + (sqid << 3);
4623 sq->ei_addr = n->dbbuf_eis + (sqid << 3);
4624
4625 if (n->params.ioeventfd && sq->sqid != 0) {
4626 if (!nvme_init_sq_ioeventfd(sq)) {
4627 sq->ioeventfd_enabled = true;
4628 }
4629 }
4630 }
4631
4632 assert(n->cq[cqid]);
4633 cq = n->cq[cqid];
4634 QTAILQ_INSERT_TAIL(&(cq->sq_list), sq, entry);
4635 n->sq[sqid] = sq;
4636 }
4637
4638 static uint16_t nvme_create_sq(NvmeCtrl *n, NvmeRequest *req)
4639 {
4640 NvmeSQueue *sq;
4641 NvmeCreateSq *c = (NvmeCreateSq *)&req->cmd;
4642
4643 uint16_t cqid = le16_to_cpu(c->cqid);
4644 uint16_t sqid = le16_to_cpu(c->sqid);
4645 uint16_t qsize = le16_to_cpu(c->qsize);
4646 uint16_t qflags = le16_to_cpu(c->sq_flags);
4647 uint64_t prp1 = le64_to_cpu(c->prp1);
4648
4649 trace_pci_nvme_create_sq(prp1, sqid, cqid, qsize, qflags);
4650
4651 if (unlikely(!cqid || nvme_check_cqid(n, cqid))) {
4652 trace_pci_nvme_err_invalid_create_sq_cqid(cqid);
4653 return NVME_INVALID_CQID | NVME_DNR;
4654 }
4655 if (unlikely(!sqid || sqid > n->conf_ioqpairs || n->sq[sqid] != NULL)) {
4656 trace_pci_nvme_err_invalid_create_sq_sqid(sqid);
4657 return NVME_INVALID_QID | NVME_DNR;
4658 }
4659 if (unlikely(!qsize || qsize > NVME_CAP_MQES(ldq_le_p(&n->bar.cap)))) {
4660 trace_pci_nvme_err_invalid_create_sq_size(qsize);
4661 return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
4662 }
4663 if (unlikely(prp1 & (n->page_size - 1))) {
4664 trace_pci_nvme_err_invalid_create_sq_addr(prp1);
4665 return NVME_INVALID_PRP_OFFSET | NVME_DNR;
4666 }
4667 if (unlikely(!(NVME_SQ_FLAGS_PC(qflags)))) {
4668 trace_pci_nvme_err_invalid_create_sq_qflags(NVME_SQ_FLAGS_PC(qflags));
4669 return NVME_INVALID_FIELD | NVME_DNR;
4670 }
4671 sq = g_malloc0(sizeof(*sq));
4672 nvme_init_sq(sq, n, prp1, sqid, cqid, qsize + 1);
4673 return NVME_SUCCESS;
4674 }
4675
4676 struct nvme_stats {
4677 uint64_t units_read;
4678 uint64_t units_written;
4679 uint64_t read_commands;
4680 uint64_t write_commands;
4681 };
4682
4683 static void nvme_set_blk_stats(NvmeNamespace *ns, struct nvme_stats *stats)
4684 {
4685 BlockAcctStats *s = blk_get_stats(ns->blkconf.blk);
4686
4687 stats->units_read += s->nr_bytes[BLOCK_ACCT_READ];
4688 stats->units_written += s->nr_bytes[BLOCK_ACCT_WRITE];
4689 stats->read_commands += s->nr_ops[BLOCK_ACCT_READ];
4690 stats->write_commands += s->nr_ops[BLOCK_ACCT_WRITE];
4691 }
4692
4693 static uint16_t nvme_smart_info(NvmeCtrl *n, uint8_t rae, uint32_t buf_len,
4694 uint64_t off, NvmeRequest *req)
4695 {
4696 uint32_t nsid = le32_to_cpu(req->cmd.nsid);
4697 struct nvme_stats stats = { 0 };
4698 NvmeSmartLog smart = { 0 };
4699 uint32_t trans_len;
4700 NvmeNamespace *ns;
4701 time_t current_ms;
4702 uint64_t u_read, u_written;
4703
4704 if (off >= sizeof(smart)) {
4705 return NVME_INVALID_FIELD | NVME_DNR;
4706 }
4707
4708 if (nsid != 0xffffffff) {
4709 ns = nvme_ns(n, nsid);
4710 if (!ns) {
4711 return NVME_INVALID_NSID | NVME_DNR;
4712 }
4713 nvme_set_blk_stats(ns, &stats);
4714 } else {
4715 int i;
4716
4717 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
4718 ns = nvme_ns(n, i);
4719 if (!ns) {
4720 continue;
4721 }
4722 nvme_set_blk_stats(ns, &stats);
4723 }
4724 }
4725
4726 trans_len = MIN(sizeof(smart) - off, buf_len);
4727 smart.critical_warning = n->smart_critical_warning;
4728
4729 u_read = DIV_ROUND_UP(stats.units_read >> BDRV_SECTOR_BITS, 1000);
4730 u_written = DIV_ROUND_UP(stats.units_written >> BDRV_SECTOR_BITS, 1000);
4731
4732 smart.data_units_read[0] = cpu_to_le64(u_read);
4733 smart.data_units_written[0] = cpu_to_le64(u_written);
4734 smart.host_read_commands[0] = cpu_to_le64(stats.read_commands);
4735 smart.host_write_commands[0] = cpu_to_le64(stats.write_commands);
4736
4737 smart.temperature = cpu_to_le16(n->temperature);
4738
4739 if ((n->temperature >= n->features.temp_thresh_hi) ||
4740 (n->temperature <= n->features.temp_thresh_low)) {
4741 smart.critical_warning |= NVME_SMART_TEMPERATURE;
4742 }
4743
4744 current_ms = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
4745 smart.power_on_hours[0] =
4746 cpu_to_le64((((current_ms - n->starttime_ms) / 1000) / 60) / 60);
4747
4748 if (!rae) {
4749 nvme_clear_events(n, NVME_AER_TYPE_SMART);
4750 }
4751
4752 return nvme_c2h(n, (uint8_t *) &smart + off, trans_len, req);
4753 }
4754
4755 static uint16_t nvme_endgrp_info(NvmeCtrl *n, uint8_t rae, uint32_t buf_len,
4756 uint64_t off, NvmeRequest *req)
4757 {
4758 uint32_t dw11 = le32_to_cpu(req->cmd.cdw11);
4759 uint16_t endgrpid = (dw11 >> 16) & 0xffff;
4760 struct nvme_stats stats = {};
4761 NvmeEndGrpLog info = {};
4762 int i;
4763
4764 if (!n->subsys || endgrpid != 0x1) {
4765 return NVME_INVALID_FIELD | NVME_DNR;
4766 }
4767
4768 if (off >= sizeof(info)) {
4769 return NVME_INVALID_FIELD | NVME_DNR;
4770 }
4771
4772 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
4773 NvmeNamespace *ns = nvme_subsys_ns(n->subsys, i);
4774 if (!ns) {
4775 continue;
4776 }
4777
4778 nvme_set_blk_stats(ns, &stats);
4779 }
4780
4781 info.data_units_read[0] =
4782 cpu_to_le64(DIV_ROUND_UP(stats.units_read / 1000000000, 1000000000));
4783 info.data_units_written[0] =
4784 cpu_to_le64(DIV_ROUND_UP(stats.units_written / 1000000000, 1000000000));
4785 info.media_units_written[0] =
4786 cpu_to_le64(DIV_ROUND_UP(stats.units_written / 1000000000, 1000000000));
4787
4788 info.host_read_commands[0] = cpu_to_le64(stats.read_commands);
4789 info.host_write_commands[0] = cpu_to_le64(stats.write_commands);
4790
4791 buf_len = MIN(sizeof(info) - off, buf_len);
4792
4793 return nvme_c2h(n, (uint8_t *)&info + off, buf_len, req);
4794 }
4795
4796
4797 static uint16_t nvme_fw_log_info(NvmeCtrl *n, uint32_t buf_len, uint64_t off,
4798 NvmeRequest *req)
4799 {
4800 uint32_t trans_len;
4801 NvmeFwSlotInfoLog fw_log = {
4802 .afi = 0x1,
4803 };
4804
4805 if (off >= sizeof(fw_log)) {
4806 return NVME_INVALID_FIELD | NVME_DNR;
4807 }
4808
4809 strpadcpy((char *)&fw_log.frs1, sizeof(fw_log.frs1), "1.0", ' ');
4810 trans_len = MIN(sizeof(fw_log) - off, buf_len);
4811
4812 return nvme_c2h(n, (uint8_t *) &fw_log + off, trans_len, req);
4813 }
4814
4815 static uint16_t nvme_error_info(NvmeCtrl *n, uint8_t rae, uint32_t buf_len,
4816 uint64_t off, NvmeRequest *req)
4817 {
4818 uint32_t trans_len;
4819 NvmeErrorLog errlog;
4820
4821 if (off >= sizeof(errlog)) {
4822 return NVME_INVALID_FIELD | NVME_DNR;
4823 }
4824
4825 if (!rae) {
4826 nvme_clear_events(n, NVME_AER_TYPE_ERROR);
4827 }
4828
4829 memset(&errlog, 0x0, sizeof(errlog));
4830 trans_len = MIN(sizeof(errlog) - off, buf_len);
4831
4832 return nvme_c2h(n, (uint8_t *)&errlog, trans_len, req);
4833 }
4834
4835 static uint16_t nvme_changed_nslist(NvmeCtrl *n, uint8_t rae, uint32_t buf_len,
4836 uint64_t off, NvmeRequest *req)
4837 {
4838 uint32_t nslist[1024];
4839 uint32_t trans_len;
4840 int i = 0;
4841 uint32_t nsid;
4842
4843 if (off >= sizeof(nslist)) {
4844 trace_pci_nvme_err_invalid_log_page_offset(off, sizeof(nslist));
4845 return NVME_INVALID_FIELD | NVME_DNR;
4846 }
4847
4848 memset(nslist, 0x0, sizeof(nslist));
4849 trans_len = MIN(sizeof(nslist) - off, buf_len);
4850
4851 while ((nsid = find_first_bit(n->changed_nsids, NVME_CHANGED_NSID_SIZE)) !=
4852 NVME_CHANGED_NSID_SIZE) {
4853 /*
4854 * If more than 1024 namespaces, the first entry in the log page should
4855 * be set to FFFFFFFFh and the others to 0 as spec.
4856 */
4857 if (i == ARRAY_SIZE(nslist)) {
4858 memset(nslist, 0x0, sizeof(nslist));
4859 nslist[0] = 0xffffffff;
4860 break;
4861 }
4862
4863 nslist[i++] = nsid;
4864 clear_bit(nsid, n->changed_nsids);
4865 }
4866
4867 /*
4868 * Remove all the remaining list entries in case returns directly due to
4869 * more than 1024 namespaces.
4870 */
4871 if (nslist[0] == 0xffffffff) {
4872 bitmap_zero(n->changed_nsids, NVME_CHANGED_NSID_SIZE);
4873 }
4874
4875 if (!rae) {
4876 nvme_clear_events(n, NVME_AER_TYPE_NOTICE);
4877 }
4878
4879 return nvme_c2h(n, ((uint8_t *)nslist) + off, trans_len, req);
4880 }
4881
4882 static uint16_t nvme_cmd_effects(NvmeCtrl *n, uint8_t csi, uint32_t buf_len,
4883 uint64_t off, NvmeRequest *req)
4884 {
4885 NvmeEffectsLog log = {};
4886 const uint32_t *src_iocs = NULL;
4887 uint32_t trans_len;
4888
4889 if (off >= sizeof(log)) {
4890 trace_pci_nvme_err_invalid_log_page_offset(off, sizeof(log));
4891 return NVME_INVALID_FIELD | NVME_DNR;
4892 }
4893
4894 switch (NVME_CC_CSS(ldl_le_p(&n->bar.cc))) {
4895 case NVME_CC_CSS_NVM:
4896 src_iocs = nvme_cse_iocs_nvm;
4897 /* fall through */
4898 case NVME_CC_CSS_ADMIN_ONLY:
4899 break;
4900 case NVME_CC_CSS_CSI:
4901 switch (csi) {
4902 case NVME_CSI_NVM:
4903 src_iocs = nvme_cse_iocs_nvm;
4904 break;
4905 case NVME_CSI_ZONED:
4906 src_iocs = nvme_cse_iocs_zoned;
4907 break;
4908 }
4909 }
4910
4911 memcpy(log.acs, nvme_cse_acs, sizeof(nvme_cse_acs));
4912
4913 if (src_iocs) {
4914 memcpy(log.iocs, src_iocs, sizeof(log.iocs));
4915 }
4916
4917 trans_len = MIN(sizeof(log) - off, buf_len);
4918
4919 return nvme_c2h(n, ((uint8_t *)&log) + off, trans_len, req);
4920 }
4921
4922 static size_t sizeof_fdp_conf_descr(size_t nruh, size_t vss)
4923 {
4924 size_t entry_siz = sizeof(NvmeFdpDescrHdr) + nruh * sizeof(NvmeRuhDescr)
4925 + vss;
4926 return ROUND_UP(entry_siz, 8);
4927 }
4928
4929 static uint16_t nvme_fdp_confs(NvmeCtrl *n, uint32_t endgrpid, uint32_t buf_len,
4930 uint64_t off, NvmeRequest *req)
4931 {
4932 uint32_t log_size, trans_len;
4933 g_autofree uint8_t *buf = NULL;
4934 NvmeFdpDescrHdr *hdr;
4935 NvmeRuhDescr *ruhd;
4936 NvmeEnduranceGroup *endgrp;
4937 NvmeFdpConfsHdr *log;
4938 size_t nruh, fdp_descr_size;
4939 int i;
4940
4941 if (endgrpid != 1 || !n->subsys) {
4942 return NVME_INVALID_FIELD | NVME_DNR;
4943 }
4944
4945 endgrp = &n->subsys->endgrp;
4946
4947 if (endgrp->fdp.enabled) {
4948 nruh = endgrp->fdp.nruh;
4949 } else {
4950 nruh = 1;
4951 }
4952
4953 fdp_descr_size = sizeof_fdp_conf_descr(nruh, FDPVSS);
4954 log_size = sizeof(NvmeFdpConfsHdr) + fdp_descr_size;
4955
4956 if (off >= log_size) {
4957 return NVME_INVALID_FIELD | NVME_DNR;
4958 }
4959
4960 trans_len = MIN(log_size - off, buf_len);
4961
4962 buf = g_malloc0(log_size);
4963 log = (NvmeFdpConfsHdr *)buf;
4964 hdr = (NvmeFdpDescrHdr *)(log + 1);
4965 ruhd = (NvmeRuhDescr *)(buf + sizeof(*log) + sizeof(*hdr));
4966
4967 log->num_confs = cpu_to_le16(0);
4968 log->size = cpu_to_le32(log_size);
4969
4970 hdr->descr_size = cpu_to_le16(fdp_descr_size);
4971 if (endgrp->fdp.enabled) {
4972 hdr->fdpa = FIELD_DP8(hdr->fdpa, FDPA, VALID, 1);
4973 hdr->fdpa = FIELD_DP8(hdr->fdpa, FDPA, RGIF, endgrp->fdp.rgif);
4974 hdr->nrg = cpu_to_le16(endgrp->fdp.nrg);
4975 hdr->nruh = cpu_to_le16(endgrp->fdp.nruh);
4976 hdr->maxpids = cpu_to_le16(NVME_FDP_MAXPIDS - 1);
4977 hdr->nnss = cpu_to_le32(NVME_MAX_NAMESPACES);
4978 hdr->runs = cpu_to_le64(endgrp->fdp.runs);
4979
4980 for (i = 0; i < nruh; i++) {
4981 ruhd->ruht = NVME_RUHT_INITIALLY_ISOLATED;
4982 ruhd++;
4983 }
4984 } else {
4985 /* 1 bit for RUH in PIF -> 2 RUHs max. */
4986 hdr->nrg = cpu_to_le16(1);
4987 hdr->nruh = cpu_to_le16(1);
4988 hdr->maxpids = cpu_to_le16(NVME_FDP_MAXPIDS - 1);
4989 hdr->nnss = cpu_to_le32(1);
4990 hdr->runs = cpu_to_le64(96 * MiB);
4991
4992 ruhd->ruht = NVME_RUHT_INITIALLY_ISOLATED;
4993 }
4994
4995 return nvme_c2h(n, (uint8_t *)buf + off, trans_len, req);
4996 }
4997
4998 static uint16_t nvme_fdp_ruh_usage(NvmeCtrl *n, uint32_t endgrpid,
4999 uint32_t dw10, uint32_t dw12,
5000 uint32_t buf_len, uint64_t off,
5001 NvmeRequest *req)
5002 {
5003 NvmeRuHandle *ruh;
5004 NvmeRuhuLog *hdr;
5005 NvmeRuhuDescr *ruhud;
5006 NvmeEnduranceGroup *endgrp;
5007 g_autofree uint8_t *buf = NULL;
5008 uint32_t log_size, trans_len;
5009 uint16_t i;
5010
5011 if (endgrpid != 1 || !n->subsys) {
5012 return NVME_INVALID_FIELD | NVME_DNR;
5013 }
5014
5015 endgrp = &n->subsys->endgrp;
5016
5017 if (!endgrp->fdp.enabled) {
5018 return NVME_FDP_DISABLED | NVME_DNR;
5019 }
5020
5021 log_size = sizeof(NvmeRuhuLog) + endgrp->fdp.nruh * sizeof(NvmeRuhuDescr);
5022
5023 if (off >= log_size) {
5024 return NVME_INVALID_FIELD | NVME_DNR;
5025 }
5026
5027 trans_len = MIN(log_size - off, buf_len);
5028
5029 buf = g_malloc0(log_size);
5030 hdr = (NvmeRuhuLog *)buf;
5031 ruhud = (NvmeRuhuDescr *)(hdr + 1);
5032
5033 ruh = endgrp->fdp.ruhs;
5034 hdr->nruh = cpu_to_le16(endgrp->fdp.nruh);
5035
5036 for (i = 0; i < endgrp->fdp.nruh; i++, ruhud++, ruh++) {
5037 ruhud->ruha = ruh->ruha;
5038 }
5039
5040 return nvme_c2h(n, (uint8_t *)buf + off, trans_len, req);
5041 }
5042
5043 static uint16_t nvme_fdp_stats(NvmeCtrl *n, uint32_t endgrpid, uint32_t buf_len,
5044 uint64_t off, NvmeRequest *req)
5045 {
5046 NvmeEnduranceGroup *endgrp;
5047 NvmeFdpStatsLog log = {};
5048 uint32_t trans_len;
5049
5050 if (off >= sizeof(NvmeFdpStatsLog)) {
5051 return NVME_INVALID_FIELD | NVME_DNR;
5052 }
5053
5054 if (endgrpid != 1 || !n->subsys) {
5055 return NVME_INVALID_FIELD | NVME_DNR;
5056 }
5057
5058 if (!n->subsys->endgrp.fdp.enabled) {
5059 return NVME_FDP_DISABLED | NVME_DNR;
5060 }
5061
5062 endgrp = &n->subsys->endgrp;
5063
5064 trans_len = MIN(sizeof(log) - off, buf_len);
5065
5066 /* spec value is 128 bit, we only use 64 bit */
5067 log.hbmw[0] = cpu_to_le64(endgrp->fdp.hbmw);
5068 log.mbmw[0] = cpu_to_le64(endgrp->fdp.mbmw);
5069 log.mbe[0] = cpu_to_le64(endgrp->fdp.mbe);
5070
5071 return nvme_c2h(n, (uint8_t *)&log + off, trans_len, req);
5072 }
5073
5074 static uint16_t nvme_fdp_events(NvmeCtrl *n, uint32_t endgrpid,
5075 uint32_t buf_len, uint64_t off,
5076 NvmeRequest *req)
5077 {
5078 NvmeEnduranceGroup *endgrp;
5079 NvmeCmd *cmd = &req->cmd;
5080 bool host_events = (cmd->cdw10 >> 8) & 0x1;
5081 uint32_t log_size, trans_len;
5082 NvmeFdpEventBuffer *ebuf;
5083 g_autofree NvmeFdpEventsLog *elog = NULL;
5084 NvmeFdpEvent *event;
5085
5086 if (endgrpid != 1 || !n->subsys) {
5087 return NVME_INVALID_FIELD | NVME_DNR;
5088 }
5089
5090 endgrp = &n->subsys->endgrp;
5091
5092 if (!endgrp->fdp.enabled) {
5093 return NVME_FDP_DISABLED | NVME_DNR;
5094 }
5095
5096 if (host_events) {
5097 ebuf = &endgrp->fdp.host_events;
5098 } else {
5099 ebuf = &endgrp->fdp.ctrl_events;
5100 }
5101
5102 log_size = sizeof(NvmeFdpEventsLog) + ebuf->nelems * sizeof(NvmeFdpEvent);
5103 trans_len = MIN(log_size - off, buf_len);
5104 elog = g_malloc0(log_size);
5105 elog->num_events = cpu_to_le32(ebuf->nelems);
5106 event = (NvmeFdpEvent *)(elog + 1);
5107
5108 if (ebuf->nelems && ebuf->start == ebuf->next) {
5109 unsigned int nelems = (NVME_FDP_MAX_EVENTS - ebuf->start);
5110 /* wrap over, copy [start;NVME_FDP_MAX_EVENTS[ and [0; next[ */
5111 memcpy(event, &ebuf->events[ebuf->start],
5112 sizeof(NvmeFdpEvent) * nelems);
5113 memcpy(event + nelems, ebuf->events,
5114 sizeof(NvmeFdpEvent) * ebuf->next);
5115 } else if (ebuf->start < ebuf->next) {
5116 memcpy(event, &ebuf->events[ebuf->start],
5117 sizeof(NvmeFdpEvent) * (ebuf->next - ebuf->start));
5118 }
5119
5120 return nvme_c2h(n, (uint8_t *)elog + off, trans_len, req);
5121 }
5122
5123 static uint16_t nvme_get_log(NvmeCtrl *n, NvmeRequest *req)
5124 {
5125 NvmeCmd *cmd = &req->cmd;
5126
5127 uint32_t dw10 = le32_to_cpu(cmd->cdw10);
5128 uint32_t dw11 = le32_to_cpu(cmd->cdw11);
5129 uint32_t dw12 = le32_to_cpu(cmd->cdw12);
5130 uint32_t dw13 = le32_to_cpu(cmd->cdw13);
5131 uint8_t lid = dw10 & 0xff;
5132 uint8_t lsp = (dw10 >> 8) & 0xf;
5133 uint8_t rae = (dw10 >> 15) & 0x1;
5134 uint8_t csi = le32_to_cpu(cmd->cdw14) >> 24;
5135 uint32_t numdl, numdu, lspi;
5136 uint64_t off, lpol, lpou;
5137 size_t len;
5138 uint16_t status;
5139
5140 numdl = (dw10 >> 16);
5141 numdu = (dw11 & 0xffff);
5142 lspi = (dw11 >> 16);
5143 lpol = dw12;
5144 lpou = dw13;
5145
5146 len = (((numdu << 16) | numdl) + 1) << 2;
5147 off = (lpou << 32ULL) | lpol;
5148
5149 if (off & 0x3) {
5150 return NVME_INVALID_FIELD | NVME_DNR;
5151 }
5152
5153 trace_pci_nvme_get_log(nvme_cid(req), lid, lsp, rae, len, off);
5154
5155 status = nvme_check_mdts(n, len);
5156 if (status) {
5157 return status;
5158 }
5159
5160 switch (lid) {
5161 case NVME_LOG_ERROR_INFO:
5162 return nvme_error_info(n, rae, len, off, req);
5163 case NVME_LOG_SMART_INFO:
5164 return nvme_smart_info(n, rae, len, off, req);
5165 case NVME_LOG_FW_SLOT_INFO:
5166 return nvme_fw_log_info(n, len, off, req);
5167 case NVME_LOG_CHANGED_NSLIST:
5168 return nvme_changed_nslist(n, rae, len, off, req);
5169 case NVME_LOG_CMD_EFFECTS:
5170 return nvme_cmd_effects(n, csi, len, off, req);
5171 case NVME_LOG_ENDGRP:
5172 return nvme_endgrp_info(n, rae, len, off, req);
5173 case NVME_LOG_FDP_CONFS:
5174 return nvme_fdp_confs(n, lspi, len, off, req);
5175 case NVME_LOG_FDP_RUH_USAGE:
5176 return nvme_fdp_ruh_usage(n, lspi, dw10, dw12, len, off, req);
5177 case NVME_LOG_FDP_STATS:
5178 return nvme_fdp_stats(n, lspi, len, off, req);
5179 case NVME_LOG_FDP_EVENTS:
5180 return nvme_fdp_events(n, lspi, len, off, req);
5181 default:
5182 trace_pci_nvme_err_invalid_log_page(nvme_cid(req), lid);
5183 return NVME_INVALID_FIELD | NVME_DNR;
5184 }
5185 }
5186
5187 static void nvme_free_cq(NvmeCQueue *cq, NvmeCtrl *n)
5188 {
5189 PCIDevice *pci = PCI_DEVICE(n);
5190 uint16_t offset = (cq->cqid << 3) + (1 << 2);
5191
5192 n->cq[cq->cqid] = NULL;
5193 qemu_bh_delete(cq->bh);
5194 if (cq->ioeventfd_enabled) {
5195 memory_region_del_eventfd(&n->iomem,
5196 0x1000 + offset, 4, false, 0, &cq->notifier);
5197 event_notifier_set_handler(&cq->notifier, NULL);
5198 event_notifier_cleanup(&cq->notifier);
5199 }
5200 if (msix_enabled(pci)) {
5201 msix_vector_unuse(pci, cq->vector);
5202 }
5203 if (cq->cqid) {
5204 g_free(cq);
5205 }
5206 }
5207
5208 static uint16_t nvme_del_cq(NvmeCtrl *n, NvmeRequest *req)
5209 {
5210 NvmeDeleteQ *c = (NvmeDeleteQ *)&req->cmd;
5211 NvmeCQueue *cq;
5212 uint16_t qid = le16_to_cpu(c->qid);
5213
5214 if (unlikely(!qid || nvme_check_cqid(n, qid))) {
5215 trace_pci_nvme_err_invalid_del_cq_cqid(qid);
5216 return NVME_INVALID_CQID | NVME_DNR;
5217 }
5218
5219 cq = n->cq[qid];
5220 if (unlikely(!QTAILQ_EMPTY(&cq->sq_list))) {
5221 trace_pci_nvme_err_invalid_del_cq_notempty(qid);
5222 return NVME_INVALID_QUEUE_DEL;
5223 }
5224
5225 if (cq->irq_enabled && cq->tail != cq->head) {
5226 n->cq_pending--;
5227 }
5228
5229 nvme_irq_deassert(n, cq);
5230 trace_pci_nvme_del_cq(qid);
5231 nvme_free_cq(cq, n);
5232 return NVME_SUCCESS;
5233 }
5234
5235 static void nvme_init_cq(NvmeCQueue *cq, NvmeCtrl *n, uint64_t dma_addr,
5236 uint16_t cqid, uint16_t vector, uint16_t size,
5237 uint16_t irq_enabled)
5238 {
5239 PCIDevice *pci = PCI_DEVICE(n);
5240
5241 if (msix_enabled(pci)) {
5242 msix_vector_use(pci, vector);
5243 }
5244 cq->ctrl = n;
5245 cq->cqid = cqid;
5246 cq->size = size;
5247 cq->dma_addr = dma_addr;
5248 cq->phase = 1;
5249 cq->irq_enabled = irq_enabled;
5250 cq->vector = vector;
5251 cq->head = cq->tail = 0;
5252 QTAILQ_INIT(&cq->req_list);
5253 QTAILQ_INIT(&cq->sq_list);
5254 if (n->dbbuf_enabled) {
5255 cq->db_addr = n->dbbuf_dbs + (cqid << 3) + (1 << 2);
5256 cq->ei_addr = n->dbbuf_eis + (cqid << 3) + (1 << 2);
5257
5258 if (n->params.ioeventfd && cqid != 0) {
5259 if (!nvme_init_cq_ioeventfd(cq)) {
5260 cq->ioeventfd_enabled = true;
5261 }
5262 }
5263 }
5264 n->cq[cqid] = cq;
5265 cq->bh = qemu_bh_new_guarded(nvme_post_cqes, cq,
5266 &DEVICE(cq->ctrl)->mem_reentrancy_guard);
5267 }
5268
5269 static uint16_t nvme_create_cq(NvmeCtrl *n, NvmeRequest *req)
5270 {
5271 NvmeCQueue *cq;
5272 NvmeCreateCq *c = (NvmeCreateCq *)&req->cmd;
5273 uint16_t cqid = le16_to_cpu(c->cqid);
5274 uint16_t vector = le16_to_cpu(c->irq_vector);
5275 uint16_t qsize = le16_to_cpu(c->qsize);
5276 uint16_t qflags = le16_to_cpu(c->cq_flags);
5277 uint64_t prp1 = le64_to_cpu(c->prp1);
5278
5279 trace_pci_nvme_create_cq(prp1, cqid, vector, qsize, qflags,
5280 NVME_CQ_FLAGS_IEN(qflags) != 0);
5281
5282 if (unlikely(!cqid || cqid > n->conf_ioqpairs || n->cq[cqid] != NULL)) {
5283 trace_pci_nvme_err_invalid_create_cq_cqid(cqid);
5284 return NVME_INVALID_QID | NVME_DNR;
5285 }
5286 if (unlikely(!qsize || qsize > NVME_CAP_MQES(ldq_le_p(&n->bar.cap)))) {
5287 trace_pci_nvme_err_invalid_create_cq_size(qsize);
5288 return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
5289 }
5290 if (unlikely(prp1 & (n->page_size - 1))) {
5291 trace_pci_nvme_err_invalid_create_cq_addr(prp1);
5292 return NVME_INVALID_PRP_OFFSET | NVME_DNR;
5293 }
5294 if (unlikely(!msix_enabled(PCI_DEVICE(n)) && vector)) {
5295 trace_pci_nvme_err_invalid_create_cq_vector(vector);
5296 return NVME_INVALID_IRQ_VECTOR | NVME_DNR;
5297 }
5298 if (unlikely(vector >= n->conf_msix_qsize)) {
5299 trace_pci_nvme_err_invalid_create_cq_vector(vector);
5300 return NVME_INVALID_IRQ_VECTOR | NVME_DNR;
5301 }
5302 if (unlikely(!(NVME_CQ_FLAGS_PC(qflags)))) {
5303 trace_pci_nvme_err_invalid_create_cq_qflags(NVME_CQ_FLAGS_PC(qflags));
5304 return NVME_INVALID_FIELD | NVME_DNR;
5305 }
5306
5307 cq = g_malloc0(sizeof(*cq));
5308 nvme_init_cq(cq, n, prp1, cqid, vector, qsize + 1,
5309 NVME_CQ_FLAGS_IEN(qflags));
5310
5311 /*
5312 * It is only required to set qs_created when creating a completion queue;
5313 * creating a submission queue without a matching completion queue will
5314 * fail.
5315 */
5316 n->qs_created = true;
5317 return NVME_SUCCESS;
5318 }
5319
5320 static uint16_t nvme_rpt_empty_id_struct(NvmeCtrl *n, NvmeRequest *req)
5321 {
5322 uint8_t id[NVME_IDENTIFY_DATA_SIZE] = {};
5323
5324 return nvme_c2h(n, id, sizeof(id), req);
5325 }
5326
5327 static uint16_t nvme_identify_ctrl(NvmeCtrl *n, NvmeRequest *req)
5328 {
5329 trace_pci_nvme_identify_ctrl();
5330
5331 return nvme_c2h(n, (uint8_t *)&n->id_ctrl, sizeof(n->id_ctrl), req);
5332 }
5333
5334 static uint16_t nvme_identify_ctrl_csi(NvmeCtrl *n, NvmeRequest *req)
5335 {
5336 NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5337 uint8_t id[NVME_IDENTIFY_DATA_SIZE] = {};
5338 NvmeIdCtrlNvm *id_nvm = (NvmeIdCtrlNvm *)&id;
5339
5340 trace_pci_nvme_identify_ctrl_csi(c->csi);
5341
5342 switch (c->csi) {
5343 case NVME_CSI_NVM:
5344 id_nvm->vsl = n->params.vsl;
5345 id_nvm->dmrsl = cpu_to_le32(n->dmrsl);
5346 break;
5347
5348 case NVME_CSI_ZONED:
5349 ((NvmeIdCtrlZoned *)&id)->zasl = n->params.zasl;
5350 break;
5351
5352 default:
5353 return NVME_INVALID_FIELD | NVME_DNR;
5354 }
5355
5356 return nvme_c2h(n, id, sizeof(id), req);
5357 }
5358
5359 static uint16_t nvme_identify_ns(NvmeCtrl *n, NvmeRequest *req, bool active)
5360 {
5361 NvmeNamespace *ns;
5362 NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5363 uint32_t nsid = le32_to_cpu(c->nsid);
5364
5365 trace_pci_nvme_identify_ns(nsid);
5366
5367 if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
5368 return NVME_INVALID_NSID | NVME_DNR;
5369 }
5370
5371 ns = nvme_ns(n, nsid);
5372 if (unlikely(!ns)) {
5373 if (!active) {
5374 ns = nvme_subsys_ns(n->subsys, nsid);
5375 if (!ns) {
5376 return nvme_rpt_empty_id_struct(n, req);
5377 }
5378 } else {
5379 return nvme_rpt_empty_id_struct(n, req);
5380 }
5381 }
5382
5383 if (active || ns->csi == NVME_CSI_NVM) {
5384 return nvme_c2h(n, (uint8_t *)&ns->id_ns, sizeof(NvmeIdNs), req);
5385 }
5386
5387 return NVME_INVALID_CMD_SET | NVME_DNR;
5388 }
5389
5390 static uint16_t nvme_identify_ctrl_list(NvmeCtrl *n, NvmeRequest *req,
5391 bool attached)
5392 {
5393 NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5394 uint32_t nsid = le32_to_cpu(c->nsid);
5395 uint16_t min_id = le16_to_cpu(c->ctrlid);
5396 uint16_t list[NVME_CONTROLLER_LIST_SIZE] = {};
5397 uint16_t *ids = &list[1];
5398 NvmeNamespace *ns;
5399 NvmeCtrl *ctrl;
5400 int cntlid, nr_ids = 0;
5401
5402 trace_pci_nvme_identify_ctrl_list(c->cns, min_id);
5403
5404 if (!n->subsys) {
5405 return NVME_INVALID_FIELD | NVME_DNR;
5406 }
5407
5408 if (attached) {
5409 if (nsid == NVME_NSID_BROADCAST) {
5410 return NVME_INVALID_FIELD | NVME_DNR;
5411 }
5412
5413 ns = nvme_subsys_ns(n->subsys, nsid);
5414 if (!ns) {
5415 return NVME_INVALID_FIELD | NVME_DNR;
5416 }
5417 }
5418
5419 for (cntlid = min_id; cntlid < ARRAY_SIZE(n->subsys->ctrls); cntlid++) {
5420 ctrl = nvme_subsys_ctrl(n->subsys, cntlid);
5421 if (!ctrl) {
5422 continue;
5423 }
5424
5425 if (attached && !nvme_ns(ctrl, nsid)) {
5426 continue;
5427 }
5428
5429 ids[nr_ids++] = cntlid;
5430 }
5431
5432 list[0] = nr_ids;
5433
5434 return nvme_c2h(n, (uint8_t *)list, sizeof(list), req);
5435 }
5436
5437 static uint16_t nvme_identify_pri_ctrl_cap(NvmeCtrl *n, NvmeRequest *req)
5438 {
5439 trace_pci_nvme_identify_pri_ctrl_cap(le16_to_cpu(n->pri_ctrl_cap.cntlid));
5440
5441 return nvme_c2h(n, (uint8_t *)&n->pri_ctrl_cap,
5442 sizeof(NvmePriCtrlCap), req);
5443 }
5444
5445 static uint16_t nvme_identify_sec_ctrl_list(NvmeCtrl *n, NvmeRequest *req)
5446 {
5447 NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5448 uint16_t pri_ctrl_id = le16_to_cpu(n->pri_ctrl_cap.cntlid);
5449 uint16_t min_id = le16_to_cpu(c->ctrlid);
5450 uint8_t num_sec_ctrl = n->sec_ctrl_list.numcntl;
5451 NvmeSecCtrlList list = {0};
5452 uint8_t i;
5453
5454 for (i = 0; i < num_sec_ctrl; i++) {
5455 if (n->sec_ctrl_list.sec[i].scid >= min_id) {
5456 list.numcntl = num_sec_ctrl - i;
5457 memcpy(&list.sec, n->sec_ctrl_list.sec + i,
5458 list.numcntl * sizeof(NvmeSecCtrlEntry));
5459 break;
5460 }
5461 }
5462
5463 trace_pci_nvme_identify_sec_ctrl_list(pri_ctrl_id, list.numcntl);
5464
5465 return nvme_c2h(n, (uint8_t *)&list, sizeof(list), req);
5466 }
5467
5468 static uint16_t nvme_identify_ns_csi(NvmeCtrl *n, NvmeRequest *req,
5469 bool active)
5470 {
5471 NvmeNamespace *ns;
5472 NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5473 uint32_t nsid = le32_to_cpu(c->nsid);
5474
5475 trace_pci_nvme_identify_ns_csi(nsid, c->csi);
5476
5477 if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
5478 return NVME_INVALID_NSID | NVME_DNR;
5479 }
5480
5481 ns = nvme_ns(n, nsid);
5482 if (unlikely(!ns)) {
5483 if (!active) {
5484 ns = nvme_subsys_ns(n->subsys, nsid);
5485 if (!ns) {
5486 return nvme_rpt_empty_id_struct(n, req);
5487 }
5488 } else {
5489 return nvme_rpt_empty_id_struct(n, req);
5490 }
5491 }
5492
5493 if (c->csi == NVME_CSI_NVM) {
5494 return nvme_c2h(n, (uint8_t *)&ns->id_ns_nvm, sizeof(NvmeIdNsNvm),
5495 req);
5496 } else if (c->csi == NVME_CSI_ZONED && ns->csi == NVME_CSI_ZONED) {
5497 return nvme_c2h(n, (uint8_t *)ns->id_ns_zoned, sizeof(NvmeIdNsZoned),
5498 req);
5499 }
5500
5501 return NVME_INVALID_FIELD | NVME_DNR;
5502 }
5503
5504 static uint16_t nvme_identify_nslist(NvmeCtrl *n, NvmeRequest *req,
5505 bool active)
5506 {
5507 NvmeNamespace *ns;
5508 NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5509 uint32_t min_nsid = le32_to_cpu(c->nsid);
5510 uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {};
5511 static const int data_len = sizeof(list);
5512 uint32_t *list_ptr = (uint32_t *)list;
5513 int i, j = 0;
5514
5515 trace_pci_nvme_identify_nslist(min_nsid);
5516
5517 /*
5518 * Both FFFFFFFFh (NVME_NSID_BROADCAST) and FFFFFFFFEh are invalid values
5519 * since the Active Namespace ID List should return namespaces with ids
5520 * *higher* than the NSID specified in the command. This is also specified
5521 * in the spec (NVM Express v1.3d, Section 5.15.4).
5522 */
5523 if (min_nsid >= NVME_NSID_BROADCAST - 1) {
5524 return NVME_INVALID_NSID | NVME_DNR;
5525 }
5526
5527 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
5528 ns = nvme_ns(n, i);
5529 if (!ns) {
5530 if (!active) {
5531 ns = nvme_subsys_ns(n->subsys, i);
5532 if (!ns) {
5533 continue;
5534 }
5535 } else {
5536 continue;
5537 }
5538 }
5539 if (ns->params.nsid <= min_nsid) {
5540 continue;
5541 }
5542 list_ptr[j++] = cpu_to_le32(ns->params.nsid);
5543 if (j == data_len / sizeof(uint32_t)) {
5544 break;
5545 }
5546 }
5547
5548 return nvme_c2h(n, list, data_len, req);
5549 }
5550
5551 static uint16_t nvme_identify_nslist_csi(NvmeCtrl *n, NvmeRequest *req,
5552 bool active)
5553 {
5554 NvmeNamespace *ns;
5555 NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5556 uint32_t min_nsid = le32_to_cpu(c->nsid);
5557 uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {};
5558 static const int data_len = sizeof(list);
5559 uint32_t *list_ptr = (uint32_t *)list;
5560 int i, j = 0;
5561
5562 trace_pci_nvme_identify_nslist_csi(min_nsid, c->csi);
5563
5564 /*
5565 * Same as in nvme_identify_nslist(), FFFFFFFFh/FFFFFFFFEh are invalid.
5566 */
5567 if (min_nsid >= NVME_NSID_BROADCAST - 1) {
5568 return NVME_INVALID_NSID | NVME_DNR;
5569 }
5570
5571 if (c->csi != NVME_CSI_NVM && c->csi != NVME_CSI_ZONED) {
5572 return NVME_INVALID_FIELD | NVME_DNR;
5573 }
5574
5575 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
5576 ns = nvme_ns(n, i);
5577 if (!ns) {
5578 if (!active) {
5579 ns = nvme_subsys_ns(n->subsys, i);
5580 if (!ns) {
5581 continue;
5582 }
5583 } else {
5584 continue;
5585 }
5586 }
5587 if (ns->params.nsid <= min_nsid || c->csi != ns->csi) {
5588 continue;
5589 }
5590 list_ptr[j++] = cpu_to_le32(ns->params.nsid);
5591 if (j == data_len / sizeof(uint32_t)) {
5592 break;
5593 }
5594 }
5595
5596 return nvme_c2h(n, list, data_len, req);
5597 }
5598
5599 static uint16_t nvme_identify_ns_descr_list(NvmeCtrl *n, NvmeRequest *req)
5600 {
5601 NvmeNamespace *ns;
5602 NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5603 uint32_t nsid = le32_to_cpu(c->nsid);
5604 uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {};
5605 uint8_t *pos = list;
5606 struct {
5607 NvmeIdNsDescr hdr;
5608 uint8_t v[NVME_NIDL_UUID];
5609 } QEMU_PACKED uuid = {};
5610 struct {
5611 NvmeIdNsDescr hdr;
5612 uint64_t v;
5613 } QEMU_PACKED eui64 = {};
5614 struct {
5615 NvmeIdNsDescr hdr;
5616 uint8_t v;
5617 } QEMU_PACKED csi = {};
5618
5619 trace_pci_nvme_identify_ns_descr_list(nsid);
5620
5621 if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
5622 return NVME_INVALID_NSID | NVME_DNR;
5623 }
5624
5625 ns = nvme_ns(n, nsid);
5626 if (unlikely(!ns)) {
5627 return NVME_INVALID_FIELD | NVME_DNR;
5628 }
5629
5630 if (!qemu_uuid_is_null(&ns->params.uuid)) {
5631 uuid.hdr.nidt = NVME_NIDT_UUID;
5632 uuid.hdr.nidl = NVME_NIDL_UUID;
5633 memcpy(uuid.v, ns->params.uuid.data, NVME_NIDL_UUID);
5634 memcpy(pos, &uuid, sizeof(uuid));
5635 pos += sizeof(uuid);
5636 }
5637
5638 if (ns->params.eui64) {
5639 eui64.hdr.nidt = NVME_NIDT_EUI64;
5640 eui64.hdr.nidl = NVME_NIDL_EUI64;
5641 eui64.v = cpu_to_be64(ns->params.eui64);
5642 memcpy(pos, &eui64, sizeof(eui64));
5643 pos += sizeof(eui64);
5644 }
5645
5646 csi.hdr.nidt = NVME_NIDT_CSI;
5647 csi.hdr.nidl = NVME_NIDL_CSI;
5648 csi.v = ns->csi;
5649 memcpy(pos, &csi, sizeof(csi));
5650 pos += sizeof(csi);
5651
5652 return nvme_c2h(n, list, sizeof(list), req);
5653 }
5654
5655 static uint16_t nvme_identify_cmd_set(NvmeCtrl *n, NvmeRequest *req)
5656 {
5657 uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {};
5658 static const int data_len = sizeof(list);
5659
5660 trace_pci_nvme_identify_cmd_set();
5661
5662 NVME_SET_CSI(*list, NVME_CSI_NVM);
5663 NVME_SET_CSI(*list, NVME_CSI_ZONED);
5664
5665 return nvme_c2h(n, list, data_len, req);
5666 }
5667
5668 static uint16_t nvme_identify(NvmeCtrl *n, NvmeRequest *req)
5669 {
5670 NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5671
5672 trace_pci_nvme_identify(nvme_cid(req), c->cns, le16_to_cpu(c->ctrlid),
5673 c->csi);
5674
5675 switch (c->cns) {
5676 case NVME_ID_CNS_NS:
5677 return nvme_identify_ns(n, req, true);
5678 case NVME_ID_CNS_NS_PRESENT:
5679 return nvme_identify_ns(n, req, false);
5680 case NVME_ID_CNS_NS_ATTACHED_CTRL_LIST:
5681 return nvme_identify_ctrl_list(n, req, true);
5682 case NVME_ID_CNS_CTRL_LIST:
5683 return nvme_identify_ctrl_list(n, req, false);
5684 case NVME_ID_CNS_PRIMARY_CTRL_CAP:
5685 return nvme_identify_pri_ctrl_cap(n, req);
5686 case NVME_ID_CNS_SECONDARY_CTRL_LIST:
5687 return nvme_identify_sec_ctrl_list(n, req);
5688 case NVME_ID_CNS_CS_NS:
5689 return nvme_identify_ns_csi(n, req, true);
5690 case NVME_ID_CNS_CS_NS_PRESENT:
5691 return nvme_identify_ns_csi(n, req, false);
5692 case NVME_ID_CNS_CTRL:
5693 return nvme_identify_ctrl(n, req);
5694 case NVME_ID_CNS_CS_CTRL:
5695 return nvme_identify_ctrl_csi(n, req);
5696 case NVME_ID_CNS_NS_ACTIVE_LIST:
5697 return nvme_identify_nslist(n, req, true);
5698 case NVME_ID_CNS_NS_PRESENT_LIST:
5699 return nvme_identify_nslist(n, req, false);
5700 case NVME_ID_CNS_CS_NS_ACTIVE_LIST:
5701 return nvme_identify_nslist_csi(n, req, true);
5702 case NVME_ID_CNS_CS_NS_PRESENT_LIST:
5703 return nvme_identify_nslist_csi(n, req, false);
5704 case NVME_ID_CNS_NS_DESCR_LIST:
5705 return nvme_identify_ns_descr_list(n, req);
5706 case NVME_ID_CNS_IO_COMMAND_SET:
5707 return nvme_identify_cmd_set(n, req);
5708 default:
5709 trace_pci_nvme_err_invalid_identify_cns(le32_to_cpu(c->cns));
5710 return NVME_INVALID_FIELD | NVME_DNR;
5711 }
5712 }
5713
5714 static uint16_t nvme_abort(NvmeCtrl *n, NvmeRequest *req)
5715 {
5716 uint16_t sqid = le32_to_cpu(req->cmd.cdw10) & 0xffff;
5717
5718 req->cqe.result = 1;
5719 if (nvme_check_sqid(n, sqid)) {
5720 return NVME_INVALID_FIELD | NVME_DNR;
5721 }
5722
5723 return NVME_SUCCESS;
5724 }
5725
5726 static inline void nvme_set_timestamp(NvmeCtrl *n, uint64_t ts)
5727 {
5728 trace_pci_nvme_setfeat_timestamp(ts);
5729
5730 n->host_timestamp = le64_to_cpu(ts);
5731 n->timestamp_set_qemu_clock_ms = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
5732 }
5733
5734 static inline uint64_t nvme_get_timestamp(const NvmeCtrl *n)
5735 {
5736 uint64_t current_time = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
5737 uint64_t elapsed_time = current_time - n->timestamp_set_qemu_clock_ms;
5738
5739 union nvme_timestamp {
5740 struct {
5741 uint64_t timestamp:48;
5742 uint64_t sync:1;
5743 uint64_t origin:3;
5744 uint64_t rsvd1:12;
5745 };
5746 uint64_t all;
5747 };
5748
5749 union nvme_timestamp ts;
5750 ts.all = 0;
5751 ts.timestamp = n->host_timestamp + elapsed_time;
5752
5753 /* If the host timestamp is non-zero, set the timestamp origin */
5754 ts.origin = n->host_timestamp ? 0x01 : 0x00;
5755
5756 trace_pci_nvme_getfeat_timestamp(ts.all);
5757
5758 return cpu_to_le64(ts.all);
5759 }
5760
5761 static uint16_t nvme_get_feature_timestamp(NvmeCtrl *n, NvmeRequest *req)
5762 {
5763 uint64_t timestamp = nvme_get_timestamp(n);
5764
5765 return nvme_c2h(n, (uint8_t *)&timestamp, sizeof(timestamp), req);
5766 }
5767
5768 static int nvme_get_feature_fdp(NvmeCtrl *n, uint32_t endgrpid,
5769 uint32_t *result)
5770 {
5771 *result = 0;
5772
5773 if (!n->subsys || !n->subsys->endgrp.fdp.enabled) {
5774 return NVME_INVALID_FIELD | NVME_DNR;
5775 }
5776
5777 *result = FIELD_DP16(0, FEAT_FDP, FDPE, 1);
5778 *result = FIELD_DP16(*result, FEAT_FDP, CONF_NDX, 0);
5779
5780 return NVME_SUCCESS;
5781 }
5782
5783 static uint16_t nvme_get_feature_fdp_events(NvmeCtrl *n, NvmeNamespace *ns,
5784 NvmeRequest *req, uint32_t *result)
5785 {
5786 NvmeCmd *cmd = &req->cmd;
5787 uint32_t cdw11 = le32_to_cpu(cmd->cdw11);
5788 uint16_t ph = cdw11 & 0xffff;
5789 uint8_t noet = (cdw11 >> 16) & 0xff;
5790 uint16_t ruhid, ret;
5791 uint32_t nentries = 0;
5792 uint8_t s_events_ndx = 0;
5793 size_t s_events_siz = sizeof(NvmeFdpEventDescr) * noet;
5794 g_autofree NvmeFdpEventDescr *s_events = g_malloc0(s_events_siz);
5795 NvmeRuHandle *ruh;
5796 NvmeFdpEventDescr *s_event;
5797
5798 if (!n->subsys || !n->subsys->endgrp.fdp.enabled) {
5799 return NVME_FDP_DISABLED | NVME_DNR;
5800 }
5801
5802 if (!nvme_ph_valid(ns, ph)) {
5803 return NVME_INVALID_FIELD | NVME_DNR;
5804 }
5805
5806 ruhid = ns->fdp.phs[ph];
5807 ruh = &n->subsys->endgrp.fdp.ruhs[ruhid];
5808
5809 assert(ruh);
5810
5811 if (unlikely(noet == 0)) {
5812 return NVME_INVALID_FIELD | NVME_DNR;
5813 }
5814
5815 for (uint8_t event_type = 0; event_type < FDP_EVT_MAX; event_type++) {
5816 uint8_t shift = nvme_fdp_evf_shifts[event_type];
5817 if (!shift && event_type) {
5818 /*
5819 * only first entry (event_type == 0) has a shift value of 0
5820 * other entries are simply unpopulated.
5821 */
5822 continue;
5823 }
5824
5825 nentries++;
5826
5827 s_event = &s_events[s_events_ndx];
5828 s_event->evt = event_type;
5829 s_event->evta = (ruh->event_filter >> shift) & 0x1;
5830
5831 /* break if all `noet` entries are filled */
5832 if ((++s_events_ndx) == noet) {
5833 break;
5834 }
5835 }
5836
5837 ret = nvme_c2h(n, s_events, s_events_siz, req);
5838 if (ret) {
5839 return ret;
5840 }
5841
5842 *result = nentries;
5843 return NVME_SUCCESS;
5844 }
5845
5846 static uint16_t nvme_get_feature(NvmeCtrl *n, NvmeRequest *req)
5847 {
5848 NvmeCmd *cmd = &req->cmd;
5849 uint32_t dw10 = le32_to_cpu(cmd->cdw10);
5850 uint32_t dw11 = le32_to_cpu(cmd->cdw11);
5851 uint32_t nsid = le32_to_cpu(cmd->nsid);
5852 uint32_t result;
5853 uint8_t fid = NVME_GETSETFEAT_FID(dw10);
5854 NvmeGetFeatureSelect sel = NVME_GETFEAT_SELECT(dw10);
5855 uint16_t iv;
5856 NvmeNamespace *ns;
5857 int i;
5858 uint16_t endgrpid = 0, ret = NVME_SUCCESS;
5859
5860 static const uint32_t nvme_feature_default[NVME_FID_MAX] = {
5861 [NVME_ARBITRATION] = NVME_ARB_AB_NOLIMIT,
5862 };
5863
5864 trace_pci_nvme_getfeat(nvme_cid(req), nsid, fid, sel, dw11);
5865
5866 if (!nvme_feature_support[fid]) {
5867 return NVME_INVALID_FIELD | NVME_DNR;
5868 }
5869
5870 if (nvme_feature_cap[fid] & NVME_FEAT_CAP_NS) {
5871 if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
5872 /*
5873 * The Reservation Notification Mask and Reservation Persistence
5874 * features require a status code of Invalid Field in Command when
5875 * NSID is FFFFFFFFh. Since the device does not support those
5876 * features we can always return Invalid Namespace or Format as we
5877 * should do for all other features.
5878 */
5879 return NVME_INVALID_NSID | NVME_DNR;
5880 }
5881
5882 if (!nvme_ns(n, nsid)) {
5883 return NVME_INVALID_FIELD | NVME_DNR;
5884 }
5885 }
5886
5887 switch (sel) {
5888 case NVME_GETFEAT_SELECT_CURRENT:
5889 break;
5890 case NVME_GETFEAT_SELECT_SAVED:
5891 /* no features are saveable by the controller; fallthrough */
5892 case NVME_GETFEAT_SELECT_DEFAULT:
5893 goto defaults;
5894 case NVME_GETFEAT_SELECT_CAP:
5895 result = nvme_feature_cap[fid];
5896 goto out;
5897 }
5898
5899 switch (fid) {
5900 case NVME_TEMPERATURE_THRESHOLD:
5901 result = 0;
5902
5903 /*
5904 * The controller only implements the Composite Temperature sensor, so
5905 * return 0 for all other sensors.
5906 */
5907 if (NVME_TEMP_TMPSEL(dw11) != NVME_TEMP_TMPSEL_COMPOSITE) {
5908 goto out;
5909 }
5910
5911 switch (NVME_TEMP_THSEL(dw11)) {
5912 case NVME_TEMP_THSEL_OVER:
5913 result = n->features.temp_thresh_hi;
5914 goto out;
5915 case NVME_TEMP_THSEL_UNDER:
5916 result = n->features.temp_thresh_low;
5917 goto out;
5918 }
5919
5920 return NVME_INVALID_FIELD | NVME_DNR;
5921 case NVME_ERROR_RECOVERY:
5922 if (!nvme_nsid_valid(n, nsid)) {
5923 return NVME_INVALID_NSID | NVME_DNR;
5924 }
5925
5926 ns = nvme_ns(n, nsid);
5927 if (unlikely(!ns)) {
5928 return NVME_INVALID_FIELD | NVME_DNR;
5929 }
5930
5931 result = ns->features.err_rec;
5932 goto out;
5933 case NVME_VOLATILE_WRITE_CACHE:
5934 result = 0;
5935 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
5936 ns = nvme_ns(n, i);
5937 if (!ns) {
5938 continue;
5939 }
5940
5941 result = blk_enable_write_cache(ns->blkconf.blk);
5942 if (result) {
5943 break;
5944 }
5945 }
5946 trace_pci_nvme_getfeat_vwcache(result ? "enabled" : "disabled");
5947 goto out;
5948 case NVME_ASYNCHRONOUS_EVENT_CONF:
5949 result = n->features.async_config;
5950 goto out;
5951 case NVME_TIMESTAMP:
5952 return nvme_get_feature_timestamp(n, req);
5953 case NVME_HOST_BEHAVIOR_SUPPORT:
5954 return nvme_c2h(n, (uint8_t *)&n->features.hbs,
5955 sizeof(n->features.hbs), req);
5956 case NVME_FDP_MODE:
5957 endgrpid = dw11 & 0xff;
5958
5959 if (endgrpid != 0x1) {
5960 return NVME_INVALID_FIELD | NVME_DNR;
5961 }
5962
5963 ret = nvme_get_feature_fdp(n, endgrpid, &result);
5964 if (ret) {
5965 return ret;
5966 }
5967 goto out;
5968 case NVME_FDP_EVENTS:
5969 if (!nvme_nsid_valid(n, nsid)) {
5970 return NVME_INVALID_NSID | NVME_DNR;
5971 }
5972
5973 ns = nvme_ns(n, nsid);
5974 if (unlikely(!ns)) {
5975 return NVME_INVALID_FIELD | NVME_DNR;
5976 }
5977
5978 ret = nvme_get_feature_fdp_events(n, ns, req, &result);
5979 if (ret) {
5980 return ret;
5981 }
5982 goto out;
5983 default:
5984 break;
5985 }
5986
5987 defaults:
5988 switch (fid) {
5989 case NVME_TEMPERATURE_THRESHOLD:
5990 result = 0;
5991
5992 if (NVME_TEMP_TMPSEL(dw11) != NVME_TEMP_TMPSEL_COMPOSITE) {
5993 break;
5994 }
5995
5996 if (NVME_TEMP_THSEL(dw11) == NVME_TEMP_THSEL_OVER) {
5997 result = NVME_TEMPERATURE_WARNING;
5998 }
5999
6000 break;
6001 case NVME_NUMBER_OF_QUEUES:
6002 result = (n->conf_ioqpairs - 1) | ((n->conf_ioqpairs - 1) << 16);
6003 trace_pci_nvme_getfeat_numq(result);
6004 break;
6005 case NVME_INTERRUPT_VECTOR_CONF:
6006 iv = dw11 & 0xffff;
6007 if (iv >= n->conf_ioqpairs + 1) {
6008 return NVME_INVALID_FIELD | NVME_DNR;
6009 }
6010
6011 result = iv;
6012 if (iv == n->admin_cq.vector) {
6013 result |= NVME_INTVC_NOCOALESCING;
6014 }
6015 break;
6016 case NVME_FDP_MODE:
6017 endgrpid = dw11 & 0xff;
6018
6019 if (endgrpid != 0x1) {
6020 return NVME_INVALID_FIELD | NVME_DNR;
6021 }
6022
6023 ret = nvme_get_feature_fdp(n, endgrpid, &result);
6024 if (ret) {
6025 return ret;
6026 }
6027 goto out;
6028
6029 break;
6030 default:
6031 result = nvme_feature_default[fid];
6032 break;
6033 }
6034
6035 out:
6036 req->cqe.result = cpu_to_le32(result);
6037 return ret;
6038 }
6039
6040 static uint16_t nvme_set_feature_timestamp(NvmeCtrl *n, NvmeRequest *req)
6041 {
6042 uint16_t ret;
6043 uint64_t timestamp;
6044
6045 ret = nvme_h2c(n, (uint8_t *)&timestamp, sizeof(timestamp), req);
6046 if (ret) {
6047 return ret;
6048 }
6049
6050 nvme_set_timestamp(n, timestamp);
6051
6052 return NVME_SUCCESS;
6053 }
6054
6055 static uint16_t nvme_set_feature_fdp_events(NvmeCtrl *n, NvmeNamespace *ns,
6056 NvmeRequest *req)
6057 {
6058 NvmeCmd *cmd = &req->cmd;
6059 uint32_t cdw11 = le32_to_cpu(cmd->cdw11);
6060 uint16_t ph = cdw11 & 0xffff;
6061 uint8_t noet = (cdw11 >> 16) & 0xff;
6062 uint16_t ret, ruhid;
6063 uint8_t enable = le32_to_cpu(cmd->cdw12) & 0x1;
6064 uint8_t event_mask = 0;
6065 unsigned int i;
6066 g_autofree uint8_t *events = g_malloc0(noet);
6067 NvmeRuHandle *ruh = NULL;
6068
6069 assert(ns);
6070
6071 if (!n->subsys || !n->subsys->endgrp.fdp.enabled) {
6072 return NVME_FDP_DISABLED | NVME_DNR;
6073 }
6074
6075 if (!nvme_ph_valid(ns, ph)) {
6076 return NVME_INVALID_FIELD | NVME_DNR;
6077 }
6078
6079 ruhid = ns->fdp.phs[ph];
6080 ruh = &n->subsys->endgrp.fdp.ruhs[ruhid];
6081
6082 ret = nvme_h2c(n, events, noet, req);
6083 if (ret) {
6084 return ret;
6085 }
6086
6087 for (i = 0; i < noet; i++) {
6088 event_mask |= (1 << nvme_fdp_evf_shifts[events[i]]);
6089 }
6090
6091 if (enable) {
6092 ruh->event_filter |= event_mask;
6093 } else {
6094 ruh->event_filter = ruh->event_filter & ~event_mask;
6095 }
6096
6097 return NVME_SUCCESS;
6098 }
6099
6100 static uint16_t nvme_set_feature(NvmeCtrl *n, NvmeRequest *req)
6101 {
6102 NvmeNamespace *ns = NULL;
6103
6104 NvmeCmd *cmd = &req->cmd;
6105 uint32_t dw10 = le32_to_cpu(cmd->cdw10);
6106 uint32_t dw11 = le32_to_cpu(cmd->cdw11);
6107 uint32_t nsid = le32_to_cpu(cmd->nsid);
6108 uint8_t fid = NVME_GETSETFEAT_FID(dw10);
6109 uint8_t save = NVME_SETFEAT_SAVE(dw10);
6110 uint16_t status;
6111 int i;
6112
6113 trace_pci_nvme_setfeat(nvme_cid(req), nsid, fid, save, dw11);
6114
6115 if (save && !(nvme_feature_cap[fid] & NVME_FEAT_CAP_SAVE)) {
6116 return NVME_FID_NOT_SAVEABLE | NVME_DNR;
6117 }
6118
6119 if (!nvme_feature_support[fid]) {
6120 return NVME_INVALID_FIELD | NVME_DNR;
6121 }
6122
6123 if (nvme_feature_cap[fid] & NVME_FEAT_CAP_NS) {
6124 if (nsid != NVME_NSID_BROADCAST) {
6125 if (!nvme_nsid_valid(n, nsid)) {
6126 return NVME_INVALID_NSID | NVME_DNR;
6127 }
6128
6129 ns = nvme_ns(n, nsid);
6130 if (unlikely(!ns)) {
6131 return NVME_INVALID_FIELD | NVME_DNR;
6132 }
6133 }
6134 } else if (nsid && nsid != NVME_NSID_BROADCAST) {
6135 if (!nvme_nsid_valid(n, nsid)) {
6136 return NVME_INVALID_NSID | NVME_DNR;
6137 }
6138
6139 return NVME_FEAT_NOT_NS_SPEC | NVME_DNR;
6140 }
6141
6142 if (!(nvme_feature_cap[fid] & NVME_FEAT_CAP_CHANGE)) {
6143 return NVME_FEAT_NOT_CHANGEABLE | NVME_DNR;
6144 }
6145
6146 switch (fid) {
6147 case NVME_TEMPERATURE_THRESHOLD:
6148 if (NVME_TEMP_TMPSEL(dw11) != NVME_TEMP_TMPSEL_COMPOSITE) {
6149 break;
6150 }
6151
6152 switch (NVME_TEMP_THSEL(dw11)) {
6153 case NVME_TEMP_THSEL_OVER:
6154 n->features.temp_thresh_hi = NVME_TEMP_TMPTH(dw11);
6155 break;
6156 case NVME_TEMP_THSEL_UNDER:
6157 n->features.temp_thresh_low = NVME_TEMP_TMPTH(dw11);
6158 break;
6159 default:
6160 return NVME_INVALID_FIELD | NVME_DNR;
6161 }
6162
6163 if ((n->temperature >= n->features.temp_thresh_hi) ||
6164 (n->temperature <= n->features.temp_thresh_low)) {
6165 nvme_smart_event(n, NVME_SMART_TEMPERATURE);
6166 }
6167
6168 break;
6169 case NVME_ERROR_RECOVERY:
6170 if (nsid == NVME_NSID_BROADCAST) {
6171 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
6172 ns = nvme_ns(n, i);
6173
6174 if (!ns) {
6175 continue;
6176 }
6177
6178 if (NVME_ID_NS_NSFEAT_DULBE(ns->id_ns.nsfeat)) {
6179 ns->features.err_rec = dw11;
6180 }
6181 }
6182
6183 break;
6184 }
6185
6186 assert(ns);
6187 if (NVME_ID_NS_NSFEAT_DULBE(ns->id_ns.nsfeat)) {
6188 ns->features.err_rec = dw11;
6189 }
6190 break;
6191 case NVME_VOLATILE_WRITE_CACHE:
6192 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
6193 ns = nvme_ns(n, i);
6194 if (!ns) {
6195 continue;
6196 }
6197
6198 if (!(dw11 & 0x1) && blk_enable_write_cache(ns->blkconf.blk)) {
6199 blk_flush(ns->blkconf.blk);
6200 }
6201
6202 blk_set_enable_write_cache(ns->blkconf.blk, dw11 & 1);
6203 }
6204
6205 break;
6206
6207 case NVME_NUMBER_OF_QUEUES:
6208 if (n->qs_created) {
6209 return NVME_CMD_SEQ_ERROR | NVME_DNR;
6210 }
6211
6212 /*
6213 * NVMe v1.3, Section 5.21.1.7: FFFFh is not an allowed value for NCQR
6214 * and NSQR.
6215 */
6216 if ((dw11 & 0xffff) == 0xffff || ((dw11 >> 16) & 0xffff) == 0xffff) {
6217 return NVME_INVALID_FIELD | NVME_DNR;
6218 }
6219
6220 trace_pci_nvme_setfeat_numq((dw11 & 0xffff) + 1,
6221 ((dw11 >> 16) & 0xffff) + 1,
6222 n->conf_ioqpairs,
6223 n->conf_ioqpairs);
6224 req->cqe.result = cpu_to_le32((n->conf_ioqpairs - 1) |
6225 ((n->conf_ioqpairs - 1) << 16));
6226 break;
6227 case NVME_ASYNCHRONOUS_EVENT_CONF:
6228 n->features.async_config = dw11;
6229 break;
6230 case NVME_TIMESTAMP:
6231 return nvme_set_feature_timestamp(n, req);
6232 case NVME_HOST_BEHAVIOR_SUPPORT:
6233 status = nvme_h2c(n, (uint8_t *)&n->features.hbs,
6234 sizeof(n->features.hbs), req);
6235 if (status) {
6236 return status;
6237 }
6238
6239 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
6240 ns = nvme_ns(n, i);
6241
6242 if (!ns) {
6243 continue;
6244 }
6245
6246 ns->id_ns.nlbaf = ns->nlbaf - 1;
6247 if (!n->features.hbs.lbafee) {
6248 ns->id_ns.nlbaf = MIN(ns->id_ns.nlbaf, 15);
6249 }
6250 }
6251
6252 return status;
6253 case NVME_COMMAND_SET_PROFILE:
6254 if (dw11 & 0x1ff) {
6255 trace_pci_nvme_err_invalid_iocsci(dw11 & 0x1ff);
6256 return NVME_CMD_SET_CMB_REJECTED | NVME_DNR;
6257 }
6258 break;
6259 case NVME_FDP_MODE:
6260 /* spec: abort with cmd seq err if there's one or more NS' in endgrp */
6261 return NVME_CMD_SEQ_ERROR | NVME_DNR;
6262 case NVME_FDP_EVENTS:
6263 return nvme_set_feature_fdp_events(n, ns, req);
6264 default:
6265 return NVME_FEAT_NOT_CHANGEABLE | NVME_DNR;
6266 }
6267 return NVME_SUCCESS;
6268 }
6269
6270 static uint16_t nvme_aer(NvmeCtrl *n, NvmeRequest *req)
6271 {
6272 trace_pci_nvme_aer(nvme_cid(req));
6273
6274 if (n->outstanding_aers > n->params.aerl) {
6275 trace_pci_nvme_aer_aerl_exceeded();
6276 return NVME_AER_LIMIT_EXCEEDED;
6277 }
6278
6279 n->aer_reqs[n->outstanding_aers] = req;
6280 n->outstanding_aers++;
6281
6282 if (!QTAILQ_EMPTY(&n->aer_queue)) {
6283 nvme_process_aers(n);
6284 }
6285
6286 return NVME_NO_COMPLETE;
6287 }
6288
6289 static void nvme_update_dmrsl(NvmeCtrl *n)
6290 {
6291 int nsid;
6292
6293 for (nsid = 1; nsid <= NVME_MAX_NAMESPACES; nsid++) {
6294 NvmeNamespace *ns = nvme_ns(n, nsid);
6295 if (!ns) {
6296 continue;
6297 }
6298
6299 n->dmrsl = MIN_NON_ZERO(n->dmrsl,
6300 BDRV_REQUEST_MAX_BYTES / nvme_l2b(ns, 1));
6301 }
6302 }
6303
6304 static void nvme_select_iocs_ns(NvmeCtrl *n, NvmeNamespace *ns)
6305 {
6306 uint32_t cc = ldl_le_p(&n->bar.cc);
6307
6308 ns->iocs = nvme_cse_iocs_none;
6309 switch (ns->csi) {
6310 case NVME_CSI_NVM:
6311 if (NVME_CC_CSS(cc) != NVME_CC_CSS_ADMIN_ONLY) {
6312 ns->iocs = nvme_cse_iocs_nvm;
6313 }
6314 break;
6315 case NVME_CSI_ZONED:
6316 if (NVME_CC_CSS(cc) == NVME_CC_CSS_CSI) {
6317 ns->iocs = nvme_cse_iocs_zoned;
6318 } else if (NVME_CC_CSS(cc) == NVME_CC_CSS_NVM) {
6319 ns->iocs = nvme_cse_iocs_nvm;
6320 }
6321 break;
6322 }
6323 }
6324
6325 static uint16_t nvme_ns_attachment(NvmeCtrl *n, NvmeRequest *req)
6326 {
6327 NvmeNamespace *ns;
6328 NvmeCtrl *ctrl;
6329 uint16_t list[NVME_CONTROLLER_LIST_SIZE] = {};
6330 uint32_t nsid = le32_to_cpu(req->cmd.nsid);
6331 uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
6332 uint8_t sel = dw10 & 0xf;
6333 uint16_t *nr_ids = &list[0];
6334 uint16_t *ids = &list[1];
6335 uint16_t ret;
6336 int i;
6337
6338 trace_pci_nvme_ns_attachment(nvme_cid(req), dw10 & 0xf);
6339
6340 if (!nvme_nsid_valid(n, nsid)) {
6341 return NVME_INVALID_NSID | NVME_DNR;
6342 }
6343
6344 ns = nvme_subsys_ns(n->subsys, nsid);
6345 if (!ns) {
6346 return NVME_INVALID_FIELD | NVME_DNR;
6347 }
6348
6349 ret = nvme_h2c(n, (uint8_t *)list, 4096, req);
6350 if (ret) {
6351 return ret;
6352 }
6353
6354 if (!*nr_ids) {
6355 return NVME_NS_CTRL_LIST_INVALID | NVME_DNR;
6356 }
6357
6358 *nr_ids = MIN(*nr_ids, NVME_CONTROLLER_LIST_SIZE - 1);
6359 for (i = 0; i < *nr_ids; i++) {
6360 ctrl = nvme_subsys_ctrl(n->subsys, ids[i]);
6361 if (!ctrl) {
6362 return NVME_NS_CTRL_LIST_INVALID | NVME_DNR;
6363 }
6364
6365 switch (sel) {
6366 case NVME_NS_ATTACHMENT_ATTACH:
6367 if (nvme_ns(ctrl, nsid)) {
6368 return NVME_NS_ALREADY_ATTACHED | NVME_DNR;
6369 }
6370
6371 if (ns->attached && !ns->params.shared) {
6372 return NVME_NS_PRIVATE | NVME_DNR;
6373 }
6374
6375 nvme_attach_ns(ctrl, ns);
6376 nvme_select_iocs_ns(ctrl, ns);
6377
6378 break;
6379
6380 case NVME_NS_ATTACHMENT_DETACH:
6381 if (!nvme_ns(ctrl, nsid)) {
6382 return NVME_NS_NOT_ATTACHED | NVME_DNR;
6383 }
6384
6385 ctrl->namespaces[nsid] = NULL;
6386 ns->attached--;
6387
6388 nvme_update_dmrsl(ctrl);
6389
6390 break;
6391
6392 default:
6393 return NVME_INVALID_FIELD | NVME_DNR;
6394 }
6395
6396 /*
6397 * Add namespace id to the changed namespace id list for event clearing
6398 * via Get Log Page command.
6399 */
6400 if (!test_and_set_bit(nsid, ctrl->changed_nsids)) {
6401 nvme_enqueue_event(ctrl, NVME_AER_TYPE_NOTICE,
6402 NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED,
6403 NVME_LOG_CHANGED_NSLIST);
6404 }
6405 }
6406
6407 return NVME_SUCCESS;
6408 }
6409
6410 typedef struct NvmeFormatAIOCB {
6411 BlockAIOCB common;
6412 BlockAIOCB *aiocb;
6413 NvmeRequest *req;
6414 int ret;
6415
6416 NvmeNamespace *ns;
6417 uint32_t nsid;
6418 bool broadcast;
6419 int64_t offset;
6420
6421 uint8_t lbaf;
6422 uint8_t mset;
6423 uint8_t pi;
6424 uint8_t pil;
6425 } NvmeFormatAIOCB;
6426
6427 static void nvme_format_cancel(BlockAIOCB *aiocb)
6428 {
6429 NvmeFormatAIOCB *iocb = container_of(aiocb, NvmeFormatAIOCB, common);
6430
6431 iocb->ret = -ECANCELED;
6432
6433 if (iocb->aiocb) {
6434 blk_aio_cancel_async(iocb->aiocb);
6435 iocb->aiocb = NULL;
6436 }
6437 }
6438
6439 static const AIOCBInfo nvme_format_aiocb_info = {
6440 .aiocb_size = sizeof(NvmeFormatAIOCB),
6441 .cancel_async = nvme_format_cancel,
6442 .get_aio_context = nvme_get_aio_context,
6443 };
6444
6445 static void nvme_format_set(NvmeNamespace *ns, uint8_t lbaf, uint8_t mset,
6446 uint8_t pi, uint8_t pil)
6447 {
6448 uint8_t lbafl = lbaf & 0xf;
6449 uint8_t lbafu = lbaf >> 4;
6450
6451 trace_pci_nvme_format_set(ns->params.nsid, lbaf, mset, pi, pil);
6452
6453 ns->id_ns.dps = (pil << 3) | pi;
6454 ns->id_ns.flbas = (lbafu << 5) | (mset << 4) | lbafl;
6455
6456 nvme_ns_init_format(ns);
6457 }
6458
6459 static void nvme_do_format(NvmeFormatAIOCB *iocb);
6460
6461 static void nvme_format_ns_cb(void *opaque, int ret)
6462 {
6463 NvmeFormatAIOCB *iocb = opaque;
6464 NvmeNamespace *ns = iocb->ns;
6465 int bytes;
6466
6467 if (iocb->ret < 0) {
6468 goto done;
6469 } else if (ret < 0) {
6470 iocb->ret = ret;
6471 goto done;
6472 }
6473
6474 assert(ns);
6475
6476 if (iocb->offset < ns->size) {
6477 bytes = MIN(BDRV_REQUEST_MAX_BYTES, ns->size - iocb->offset);
6478
6479 iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk, iocb->offset,
6480 bytes, BDRV_REQ_MAY_UNMAP,
6481 nvme_format_ns_cb, iocb);
6482
6483 iocb->offset += bytes;
6484 return;
6485 }
6486
6487 nvme_format_set(ns, iocb->lbaf, iocb->mset, iocb->pi, iocb->pil);
6488 ns->status = 0x0;
6489 iocb->ns = NULL;
6490 iocb->offset = 0;
6491
6492 done:
6493 nvme_do_format(iocb);
6494 }
6495
6496 static uint16_t nvme_format_check(NvmeNamespace *ns, uint8_t lbaf, uint8_t pi)
6497 {
6498 if (ns->params.zoned) {
6499 return NVME_INVALID_FORMAT | NVME_DNR;
6500 }
6501
6502 if (lbaf > ns->id_ns.nlbaf) {
6503 return NVME_INVALID_FORMAT | NVME_DNR;
6504 }
6505
6506 if (pi && (ns->id_ns.lbaf[lbaf].ms < nvme_pi_tuple_size(ns))) {
6507 return NVME_INVALID_FORMAT | NVME_DNR;
6508 }
6509
6510 if (pi && pi > NVME_ID_NS_DPS_TYPE_3) {
6511 return NVME_INVALID_FIELD | NVME_DNR;
6512 }
6513
6514 return NVME_SUCCESS;
6515 }
6516
6517 static void nvme_do_format(NvmeFormatAIOCB *iocb)
6518 {
6519 NvmeRequest *req = iocb->req;
6520 NvmeCtrl *n = nvme_ctrl(req);
6521 uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
6522 uint8_t lbaf = dw10 & 0xf;
6523 uint8_t pi = (dw10 >> 5) & 0x7;
6524 uint16_t status;
6525 int i;
6526
6527 if (iocb->ret < 0) {
6528 goto done;
6529 }
6530
6531 if (iocb->broadcast) {
6532 for (i = iocb->nsid + 1; i <= NVME_MAX_NAMESPACES; i++) {
6533 iocb->ns = nvme_ns(n, i);
6534 if (iocb->ns) {
6535 iocb->nsid = i;
6536 break;
6537 }
6538 }
6539 }
6540
6541 if (!iocb->ns) {
6542 goto done;
6543 }
6544
6545 status = nvme_format_check(iocb->ns, lbaf, pi);
6546 if (status) {
6547 req->status = status;
6548 goto done;
6549 }
6550
6551 iocb->ns->status = NVME_FORMAT_IN_PROGRESS;
6552 nvme_format_ns_cb(iocb, 0);
6553 return;
6554
6555 done:
6556 iocb->common.cb(iocb->common.opaque, iocb->ret);
6557 qemu_aio_unref(iocb);
6558 }
6559
6560 static uint16_t nvme_format(NvmeCtrl *n, NvmeRequest *req)
6561 {
6562 NvmeFormatAIOCB *iocb;
6563 uint32_t nsid = le32_to_cpu(req->cmd.nsid);
6564 uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
6565 uint8_t lbaf = dw10 & 0xf;
6566 uint8_t mset = (dw10 >> 4) & 0x1;
6567 uint8_t pi = (dw10 >> 5) & 0x7;
6568 uint8_t pil = (dw10 >> 8) & 0x1;
6569 uint8_t lbafu = (dw10 >> 12) & 0x3;
6570 uint16_t status;
6571
6572 iocb = qemu_aio_get(&nvme_format_aiocb_info, NULL, nvme_misc_cb, req);
6573
6574 iocb->req = req;
6575 iocb->ret = 0;
6576 iocb->ns = NULL;
6577 iocb->nsid = 0;
6578 iocb->lbaf = lbaf;
6579 iocb->mset = mset;
6580 iocb->pi = pi;
6581 iocb->pil = pil;
6582 iocb->broadcast = (nsid == NVME_NSID_BROADCAST);
6583 iocb->offset = 0;
6584
6585 if (n->features.hbs.lbafee) {
6586 iocb->lbaf |= lbafu << 4;
6587 }
6588
6589 if (!iocb->broadcast) {
6590 if (!nvme_nsid_valid(n, nsid)) {
6591 status = NVME_INVALID_NSID | NVME_DNR;
6592 goto out;
6593 }
6594
6595 iocb->ns = nvme_ns(n, nsid);
6596 if (!iocb->ns) {
6597 status = NVME_INVALID_FIELD | NVME_DNR;
6598 goto out;
6599 }
6600 }
6601
6602 req->aiocb = &iocb->common;
6603 nvme_do_format(iocb);
6604
6605 return NVME_NO_COMPLETE;
6606
6607 out:
6608 qemu_aio_unref(iocb);
6609
6610 return status;
6611 }
6612
6613 static void nvme_get_virt_res_num(NvmeCtrl *n, uint8_t rt, int *num_total,
6614 int *num_prim, int *num_sec)
6615 {
6616 *num_total = le32_to_cpu(rt ?
6617 n->pri_ctrl_cap.vifrt : n->pri_ctrl_cap.vqfrt);
6618 *num_prim = le16_to_cpu(rt ?
6619 n->pri_ctrl_cap.virfap : n->pri_ctrl_cap.vqrfap);
6620 *num_sec = le16_to_cpu(rt ? n->pri_ctrl_cap.virfa : n->pri_ctrl_cap.vqrfa);
6621 }
6622
6623 static uint16_t nvme_assign_virt_res_to_prim(NvmeCtrl *n, NvmeRequest *req,
6624 uint16_t cntlid, uint8_t rt,
6625 int nr)
6626 {
6627 int num_total, num_prim, num_sec;
6628
6629 if (cntlid != n->cntlid) {
6630 return NVME_INVALID_CTRL_ID | NVME_DNR;
6631 }
6632
6633 nvme_get_virt_res_num(n, rt, &num_total, &num_prim, &num_sec);
6634
6635 if (nr > num_total) {
6636 return NVME_INVALID_NUM_RESOURCES | NVME_DNR;
6637 }
6638
6639 if (nr > num_total - num_sec) {
6640 return NVME_INVALID_RESOURCE_ID | NVME_DNR;
6641 }
6642
6643 if (rt) {
6644 n->next_pri_ctrl_cap.virfap = cpu_to_le16(nr);
6645 } else {
6646 n->next_pri_ctrl_cap.vqrfap = cpu_to_le16(nr);
6647 }
6648
6649 req->cqe.result = cpu_to_le32(nr);
6650 return req->status;
6651 }
6652
6653 static void nvme_update_virt_res(NvmeCtrl *n, NvmeSecCtrlEntry *sctrl,
6654 uint8_t rt, int nr)
6655 {
6656 int prev_nr, prev_total;
6657
6658 if (rt) {
6659 prev_nr = le16_to_cpu(sctrl->nvi);
6660 prev_total = le32_to_cpu(n->pri_ctrl_cap.virfa);
6661 sctrl->nvi = cpu_to_le16(nr);
6662 n->pri_ctrl_cap.virfa = cpu_to_le32(prev_total + nr - prev_nr);
6663 } else {
6664 prev_nr = le16_to_cpu(sctrl->nvq);
6665 prev_total = le32_to_cpu(n->pri_ctrl_cap.vqrfa);
6666 sctrl->nvq = cpu_to_le16(nr);
6667 n->pri_ctrl_cap.vqrfa = cpu_to_le32(prev_total + nr - prev_nr);
6668 }
6669 }
6670
6671 static uint16_t nvme_assign_virt_res_to_sec(NvmeCtrl *n, NvmeRequest *req,
6672 uint16_t cntlid, uint8_t rt, int nr)
6673 {
6674 int num_total, num_prim, num_sec, num_free, diff, limit;
6675 NvmeSecCtrlEntry *sctrl;
6676
6677 sctrl = nvme_sctrl_for_cntlid(n, cntlid);
6678 if (!sctrl) {
6679 return NVME_INVALID_CTRL_ID | NVME_DNR;
6680 }
6681
6682 if (sctrl->scs) {
6683 return NVME_INVALID_SEC_CTRL_STATE | NVME_DNR;
6684 }
6685
6686 limit = le16_to_cpu(rt ? n->pri_ctrl_cap.vifrsm : n->pri_ctrl_cap.vqfrsm);
6687 if (nr > limit) {
6688 return NVME_INVALID_NUM_RESOURCES | NVME_DNR;
6689 }
6690
6691 nvme_get_virt_res_num(n, rt, &num_total, &num_prim, &num_sec);
6692 num_free = num_total - num_prim - num_sec;
6693 diff = nr - le16_to_cpu(rt ? sctrl->nvi : sctrl->nvq);
6694
6695 if (diff > num_free) {
6696 return NVME_INVALID_RESOURCE_ID | NVME_DNR;
6697 }
6698
6699 nvme_update_virt_res(n, sctrl, rt, nr);
6700 req->cqe.result = cpu_to_le32(nr);
6701
6702 return req->status;
6703 }
6704
6705 static uint16_t nvme_virt_set_state(NvmeCtrl *n, uint16_t cntlid, bool online)
6706 {
6707 PCIDevice *pci = PCI_DEVICE(n);
6708 NvmeCtrl *sn = NULL;
6709 NvmeSecCtrlEntry *sctrl;
6710 int vf_index;
6711
6712 sctrl = nvme_sctrl_for_cntlid(n, cntlid);
6713 if (!sctrl) {
6714 return NVME_INVALID_CTRL_ID | NVME_DNR;
6715 }
6716
6717 if (!pci_is_vf(pci)) {
6718 vf_index = le16_to_cpu(sctrl->vfn) - 1;
6719 sn = NVME(pcie_sriov_get_vf_at_index(pci, vf_index));
6720 }
6721
6722 if (online) {
6723 if (!sctrl->nvi || (le16_to_cpu(sctrl->nvq) < 2) || !sn) {
6724 return NVME_INVALID_SEC_CTRL_STATE | NVME_DNR;
6725 }
6726
6727 if (!sctrl->scs) {
6728 sctrl->scs = 0x1;
6729 nvme_ctrl_reset(sn, NVME_RESET_FUNCTION);
6730 }
6731 } else {
6732 nvme_update_virt_res(n, sctrl, NVME_VIRT_RES_INTERRUPT, 0);
6733 nvme_update_virt_res(n, sctrl, NVME_VIRT_RES_QUEUE, 0);
6734
6735 if (sctrl->scs) {
6736 sctrl->scs = 0x0;
6737 if (sn) {
6738 nvme_ctrl_reset(sn, NVME_RESET_FUNCTION);
6739 }
6740 }
6741 }
6742
6743 return NVME_SUCCESS;
6744 }
6745
6746 static uint16_t nvme_virt_mngmt(NvmeCtrl *n, NvmeRequest *req)
6747 {
6748 uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
6749 uint32_t dw11 = le32_to_cpu(req->cmd.cdw11);
6750 uint8_t act = dw10 & 0xf;
6751 uint8_t rt = (dw10 >> 8) & 0x7;
6752 uint16_t cntlid = (dw10 >> 16) & 0xffff;
6753 int nr = dw11 & 0xffff;
6754
6755 trace_pci_nvme_virt_mngmt(nvme_cid(req), act, cntlid, rt ? "VI" : "VQ", nr);
6756
6757 if (rt != NVME_VIRT_RES_QUEUE && rt != NVME_VIRT_RES_INTERRUPT) {
6758 return NVME_INVALID_RESOURCE_ID | NVME_DNR;
6759 }
6760
6761 switch (act) {
6762 case NVME_VIRT_MNGMT_ACTION_SEC_ASSIGN:
6763 return nvme_assign_virt_res_to_sec(n, req, cntlid, rt, nr);
6764 case NVME_VIRT_MNGMT_ACTION_PRM_ALLOC:
6765 return nvme_assign_virt_res_to_prim(n, req, cntlid, rt, nr);
6766 case NVME_VIRT_MNGMT_ACTION_SEC_ONLINE:
6767 return nvme_virt_set_state(n, cntlid, true);
6768 case NVME_VIRT_MNGMT_ACTION_SEC_OFFLINE:
6769 return nvme_virt_set_state(n, cntlid, false);
6770 default:
6771 return NVME_INVALID_FIELD | NVME_DNR;
6772 }
6773 }
6774
6775 static uint16_t nvme_dbbuf_config(NvmeCtrl *n, const NvmeRequest *req)
6776 {
6777 PCIDevice *pci = PCI_DEVICE(n);
6778 uint64_t dbs_addr = le64_to_cpu(req->cmd.dptr.prp1);
6779 uint64_t eis_addr = le64_to_cpu(req->cmd.dptr.prp2);
6780 int i;
6781
6782 /* Address should be page aligned */
6783 if (dbs_addr & (n->page_size - 1) || eis_addr & (n->page_size - 1)) {
6784 return NVME_INVALID_FIELD | NVME_DNR;
6785 }
6786
6787 /* Save shadow buffer base addr for use during queue creation */
6788 n->dbbuf_dbs = dbs_addr;
6789 n->dbbuf_eis = eis_addr;
6790 n->dbbuf_enabled = true;
6791
6792 for (i = 0; i < n->params.max_ioqpairs + 1; i++) {
6793 NvmeSQueue *sq = n->sq[i];
6794 NvmeCQueue *cq = n->cq[i];
6795
6796 if (sq) {
6797 /*
6798 * CAP.DSTRD is 0, so offset of ith sq db_addr is (i<<3)
6799 * nvme_process_db() uses this hard-coded way to calculate
6800 * doorbell offsets. Be consistent with that here.
6801 */
6802 sq->db_addr = dbs_addr + (i << 3);
6803 sq->ei_addr = eis_addr + (i << 3);
6804 pci_dma_write(pci, sq->db_addr, &sq->tail, sizeof(sq->tail));
6805
6806 if (n->params.ioeventfd && sq->sqid != 0) {
6807 if (!nvme_init_sq_ioeventfd(sq)) {
6808 sq->ioeventfd_enabled = true;
6809 }
6810 }
6811 }
6812
6813 if (cq) {
6814 /* CAP.DSTRD is 0, so offset of ith cq db_addr is (i<<3)+(1<<2) */
6815 cq->db_addr = dbs_addr + (i << 3) + (1 << 2);
6816 cq->ei_addr = eis_addr + (i << 3) + (1 << 2);
6817 pci_dma_write(pci, cq->db_addr, &cq->head, sizeof(cq->head));
6818
6819 if (n->params.ioeventfd && cq->cqid != 0) {
6820 if (!nvme_init_cq_ioeventfd(cq)) {
6821 cq->ioeventfd_enabled = true;
6822 }
6823 }
6824 }
6825 }
6826
6827 trace_pci_nvme_dbbuf_config(dbs_addr, eis_addr);
6828
6829 return NVME_SUCCESS;
6830 }
6831
6832 static uint16_t nvme_directive_send(NvmeCtrl *n, NvmeRequest *req)
6833 {
6834 return NVME_INVALID_FIELD | NVME_DNR;
6835 }
6836
6837 static uint16_t nvme_directive_receive(NvmeCtrl *n, NvmeRequest *req)
6838 {
6839 NvmeNamespace *ns;
6840 uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
6841 uint32_t dw11 = le32_to_cpu(req->cmd.cdw11);
6842 uint32_t nsid = le32_to_cpu(req->cmd.nsid);
6843 uint8_t doper, dtype;
6844 uint32_t numd, trans_len;
6845 NvmeDirectiveIdentify id = {
6846 .supported = 1 << NVME_DIRECTIVE_IDENTIFY,
6847 .enabled = 1 << NVME_DIRECTIVE_IDENTIFY,
6848 };
6849
6850 numd = dw10 + 1;
6851 doper = dw11 & 0xff;
6852 dtype = (dw11 >> 8) & 0xff;
6853
6854 trans_len = MIN(sizeof(NvmeDirectiveIdentify), numd << 2);
6855
6856 if (nsid == NVME_NSID_BROADCAST || dtype != NVME_DIRECTIVE_IDENTIFY ||
6857 doper != NVME_DIRECTIVE_RETURN_PARAMS) {
6858 return NVME_INVALID_FIELD | NVME_DNR;
6859 }
6860
6861 ns = nvme_ns(n, nsid);
6862 if (!ns) {
6863 return NVME_INVALID_FIELD | NVME_DNR;
6864 }
6865
6866 switch (dtype) {
6867 case NVME_DIRECTIVE_IDENTIFY:
6868 switch (doper) {
6869 case NVME_DIRECTIVE_RETURN_PARAMS:
6870 if (ns->endgrp->fdp.enabled) {
6871 id.supported |= 1 << NVME_DIRECTIVE_DATA_PLACEMENT;
6872 id.enabled |= 1 << NVME_DIRECTIVE_DATA_PLACEMENT;
6873 id.persistent |= 1 << NVME_DIRECTIVE_DATA_PLACEMENT;
6874 }
6875
6876 return nvme_c2h(n, (uint8_t *)&id, trans_len, req);
6877
6878 default:
6879 return NVME_INVALID_FIELD | NVME_DNR;
6880 }
6881
6882 default:
6883 return NVME_INVALID_FIELD;
6884 }
6885 }
6886
6887 static uint16_t nvme_admin_cmd(NvmeCtrl *n, NvmeRequest *req)
6888 {
6889 trace_pci_nvme_admin_cmd(nvme_cid(req), nvme_sqid(req), req->cmd.opcode,
6890 nvme_adm_opc_str(req->cmd.opcode));
6891
6892 if (!(nvme_cse_acs[req->cmd.opcode] & NVME_CMD_EFF_CSUPP)) {
6893 trace_pci_nvme_err_invalid_admin_opc(req->cmd.opcode);
6894 return NVME_INVALID_OPCODE | NVME_DNR;
6895 }
6896
6897 /* SGLs shall not be used for Admin commands in NVMe over PCIe */
6898 if (NVME_CMD_FLAGS_PSDT(req->cmd.flags) != NVME_PSDT_PRP) {
6899 return NVME_INVALID_FIELD | NVME_DNR;
6900 }
6901
6902 if (NVME_CMD_FLAGS_FUSE(req->cmd.flags)) {
6903 return NVME_INVALID_FIELD;
6904 }
6905
6906 switch (req->cmd.opcode) {
6907 case NVME_ADM_CMD_DELETE_SQ:
6908 return nvme_del_sq(n, req);
6909 case NVME_ADM_CMD_CREATE_SQ:
6910 return nvme_create_sq(n, req);
6911 case NVME_ADM_CMD_GET_LOG_PAGE:
6912 return nvme_get_log(n, req);
6913 case NVME_ADM_CMD_DELETE_CQ:
6914 return nvme_del_cq(n, req);
6915 case NVME_ADM_CMD_CREATE_CQ:
6916 return nvme_create_cq(n, req);
6917 case NVME_ADM_CMD_IDENTIFY:
6918 return nvme_identify(n, req);
6919 case NVME_ADM_CMD_ABORT:
6920 return nvme_abort(n, req);
6921 case NVME_ADM_CMD_SET_FEATURES:
6922 return nvme_set_feature(n, req);
6923 case NVME_ADM_CMD_GET_FEATURES:
6924 return nvme_get_feature(n, req);
6925 case NVME_ADM_CMD_ASYNC_EV_REQ:
6926 return nvme_aer(n, req);
6927 case NVME_ADM_CMD_NS_ATTACHMENT:
6928 return nvme_ns_attachment(n, req);
6929 case NVME_ADM_CMD_VIRT_MNGMT:
6930 return nvme_virt_mngmt(n, req);
6931 case NVME_ADM_CMD_DBBUF_CONFIG:
6932 return nvme_dbbuf_config(n, req);
6933 case NVME_ADM_CMD_FORMAT_NVM:
6934 return nvme_format(n, req);
6935 case NVME_ADM_CMD_DIRECTIVE_SEND:
6936 return nvme_directive_send(n, req);
6937 case NVME_ADM_CMD_DIRECTIVE_RECV:
6938 return nvme_directive_receive(n, req);
6939 default:
6940 assert(false);
6941 }
6942
6943 return NVME_INVALID_OPCODE | NVME_DNR;
6944 }
6945
6946 static void nvme_update_sq_eventidx(const NvmeSQueue *sq)
6947 {
6948 uint32_t v = cpu_to_le32(sq->tail);
6949
6950 trace_pci_nvme_update_sq_eventidx(sq->sqid, sq->tail);
6951
6952 pci_dma_write(PCI_DEVICE(sq->ctrl), sq->ei_addr, &v, sizeof(v));
6953 }
6954
6955 static void nvme_update_sq_tail(NvmeSQueue *sq)
6956 {
6957 uint32_t v;
6958
6959 pci_dma_read(PCI_DEVICE(sq->ctrl), sq->db_addr, &v, sizeof(v));
6960
6961 sq->tail = le32_to_cpu(v);
6962
6963 trace_pci_nvme_update_sq_tail(sq->sqid, sq->tail);
6964 }
6965
6966 static void nvme_process_sq(void *opaque)
6967 {
6968 NvmeSQueue *sq = opaque;
6969 NvmeCtrl *n = sq->ctrl;
6970 NvmeCQueue *cq = n->cq[sq->cqid];
6971
6972 uint16_t status;
6973 hwaddr addr;
6974 NvmeCmd cmd;
6975 NvmeRequest *req;
6976
6977 if (n->dbbuf_enabled) {
6978 nvme_update_sq_tail(sq);
6979 }
6980
6981 while (!(nvme_sq_empty(sq) || QTAILQ_EMPTY(&sq->req_list))) {
6982 addr = sq->dma_addr + sq->head * n->sqe_size;
6983 if (nvme_addr_read(n, addr, (void *)&cmd, sizeof(cmd))) {
6984 trace_pci_nvme_err_addr_read(addr);
6985 trace_pci_nvme_err_cfs();
6986 stl_le_p(&n->bar.csts, NVME_CSTS_FAILED);
6987 break;
6988 }
6989 nvme_inc_sq_head(sq);
6990
6991 req = QTAILQ_FIRST(&sq->req_list);
6992 QTAILQ_REMOVE(&sq->req_list, req, entry);
6993 QTAILQ_INSERT_TAIL(&sq->out_req_list, req, entry);
6994 nvme_req_clear(req);
6995 req->cqe.cid = cmd.cid;
6996 memcpy(&req->cmd, &cmd, sizeof(NvmeCmd));
6997
6998 status = sq->sqid ? nvme_io_cmd(n, req) :
6999 nvme_admin_cmd(n, req);
7000 if (status != NVME_NO_COMPLETE) {
7001 req->status = status;
7002 nvme_enqueue_req_completion(cq, req);
7003 }
7004
7005 if (n->dbbuf_enabled) {
7006 nvme_update_sq_eventidx(sq);
7007 nvme_update_sq_tail(sq);
7008 }
7009 }
7010 }
7011
7012 static void nvme_update_msixcap_ts(PCIDevice *pci_dev, uint32_t table_size)
7013 {
7014 uint8_t *config;
7015
7016 if (!msix_present(pci_dev)) {
7017 return;
7018 }
7019
7020 assert(table_size > 0 && table_size <= pci_dev->msix_entries_nr);
7021
7022 config = pci_dev->config + pci_dev->msix_cap;
7023 pci_set_word_by_mask(config + PCI_MSIX_FLAGS, PCI_MSIX_FLAGS_QSIZE,
7024 table_size - 1);
7025 }
7026
7027 static void nvme_activate_virt_res(NvmeCtrl *n)
7028 {
7029 PCIDevice *pci_dev = PCI_DEVICE(n);
7030 NvmePriCtrlCap *cap = &n->pri_ctrl_cap;
7031 NvmeSecCtrlEntry *sctrl;
7032
7033 /* -1 to account for the admin queue */
7034 if (pci_is_vf(pci_dev)) {
7035 sctrl = nvme_sctrl(n);
7036 cap->vqprt = sctrl->nvq;
7037 cap->viprt = sctrl->nvi;
7038 n->conf_ioqpairs = sctrl->nvq ? le16_to_cpu(sctrl->nvq) - 1 : 0;
7039 n->conf_msix_qsize = sctrl->nvi ? le16_to_cpu(sctrl->nvi) : 1;
7040 } else {
7041 cap->vqrfap = n->next_pri_ctrl_cap.vqrfap;
7042 cap->virfap = n->next_pri_ctrl_cap.virfap;
7043 n->conf_ioqpairs = le16_to_cpu(cap->vqprt) +
7044 le16_to_cpu(cap->vqrfap) - 1;
7045 n->conf_msix_qsize = le16_to_cpu(cap->viprt) +
7046 le16_to_cpu(cap->virfap);
7047 }
7048 }
7049
7050 static void nvme_ctrl_reset(NvmeCtrl *n, NvmeResetType rst)
7051 {
7052 PCIDevice *pci_dev = PCI_DEVICE(n);
7053 NvmeSecCtrlEntry *sctrl;
7054 NvmeNamespace *ns;
7055 int i;
7056
7057 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
7058 ns = nvme_ns(n, i);
7059 if (!ns) {
7060 continue;
7061 }
7062
7063 nvme_ns_drain(ns);
7064 }
7065
7066 for (i = 0; i < n->params.max_ioqpairs + 1; i++) {
7067 if (n->sq[i] != NULL) {
7068 nvme_free_sq(n->sq[i], n);
7069 }
7070 }
7071 for (i = 0; i < n->params.max_ioqpairs + 1; i++) {
7072 if (n->cq[i] != NULL) {
7073 nvme_free_cq(n->cq[i], n);
7074 }
7075 }
7076
7077 while (!QTAILQ_EMPTY(&n->aer_queue)) {
7078 NvmeAsyncEvent *event = QTAILQ_FIRST(&n->aer_queue);
7079 QTAILQ_REMOVE(&n->aer_queue, event, entry);
7080 g_free(event);
7081 }
7082
7083 if (n->params.sriov_max_vfs) {
7084 if (!pci_is_vf(pci_dev)) {
7085 for (i = 0; i < n->sec_ctrl_list.numcntl; i++) {
7086 sctrl = &n->sec_ctrl_list.sec[i];
7087 nvme_virt_set_state(n, le16_to_cpu(sctrl->scid), false);
7088 }
7089
7090 if (rst != NVME_RESET_CONTROLLER) {
7091 pcie_sriov_pf_disable_vfs(pci_dev);
7092 }
7093 }
7094
7095 if (rst != NVME_RESET_CONTROLLER) {
7096 nvme_activate_virt_res(n);
7097 }
7098 }
7099
7100 n->aer_queued = 0;
7101 n->aer_mask = 0;
7102 n->outstanding_aers = 0;
7103 n->qs_created = false;
7104
7105 nvme_update_msixcap_ts(pci_dev, n->conf_msix_qsize);
7106
7107 if (pci_is_vf(pci_dev)) {
7108 sctrl = nvme_sctrl(n);
7109
7110 stl_le_p(&n->bar.csts, sctrl->scs ? 0 : NVME_CSTS_FAILED);
7111 } else {
7112 stl_le_p(&n->bar.csts, 0);
7113 }
7114
7115 stl_le_p(&n->bar.intms, 0);
7116 stl_le_p(&n->bar.intmc, 0);
7117 stl_le_p(&n->bar.cc, 0);
7118
7119 n->dbbuf_dbs = 0;
7120 n->dbbuf_eis = 0;
7121 n->dbbuf_enabled = false;
7122 }
7123
7124 static void nvme_ctrl_shutdown(NvmeCtrl *n)
7125 {
7126 NvmeNamespace *ns;
7127 int i;
7128
7129 if (n->pmr.dev) {
7130 memory_region_msync(&n->pmr.dev->mr, 0, n->pmr.dev->size);
7131 }
7132
7133 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
7134 ns = nvme_ns(n, i);
7135 if (!ns) {
7136 continue;
7137 }
7138
7139 nvme_ns_shutdown(ns);
7140 }
7141 }
7142
7143 static void nvme_select_iocs(NvmeCtrl *n)
7144 {
7145 NvmeNamespace *ns;
7146 int i;
7147
7148 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
7149 ns = nvme_ns(n, i);
7150 if (!ns) {
7151 continue;
7152 }
7153
7154 nvme_select_iocs_ns(n, ns);
7155 }
7156 }
7157
7158 static int nvme_start_ctrl(NvmeCtrl *n)
7159 {
7160 uint64_t cap = ldq_le_p(&n->bar.cap);
7161 uint32_t cc = ldl_le_p(&n->bar.cc);
7162 uint32_t aqa = ldl_le_p(&n->bar.aqa);
7163 uint64_t asq = ldq_le_p(&n->bar.asq);
7164 uint64_t acq = ldq_le_p(&n->bar.acq);
7165 uint32_t page_bits = NVME_CC_MPS(cc) + 12;
7166 uint32_t page_size = 1 << page_bits;
7167 NvmeSecCtrlEntry *sctrl = nvme_sctrl(n);
7168
7169 if (pci_is_vf(PCI_DEVICE(n)) && !sctrl->scs) {
7170 trace_pci_nvme_err_startfail_virt_state(le16_to_cpu(sctrl->nvi),
7171 le16_to_cpu(sctrl->nvq));
7172 return -1;
7173 }
7174 if (unlikely(n->cq[0])) {
7175 trace_pci_nvme_err_startfail_cq();
7176 return -1;
7177 }
7178 if (unlikely(n->sq[0])) {
7179 trace_pci_nvme_err_startfail_sq();
7180 return -1;
7181 }
7182 if (unlikely(asq & (page_size - 1))) {
7183 trace_pci_nvme_err_startfail_asq_misaligned(asq);
7184 return -1;
7185 }
7186 if (unlikely(acq & (page_size - 1))) {
7187 trace_pci_nvme_err_startfail_acq_misaligned(acq);
7188 return -1;
7189 }
7190 if (unlikely(!(NVME_CAP_CSS(cap) & (1 << NVME_CC_CSS(cc))))) {
7191 trace_pci_nvme_err_startfail_css(NVME_CC_CSS(cc));
7192 return -1;
7193 }
7194 if (unlikely(NVME_CC_MPS(cc) < NVME_CAP_MPSMIN(cap))) {
7195 trace_pci_nvme_err_startfail_page_too_small(
7196 NVME_CC_MPS(cc),
7197 NVME_CAP_MPSMIN(cap));
7198 return -1;
7199 }
7200 if (unlikely(NVME_CC_MPS(cc) >
7201 NVME_CAP_MPSMAX(cap))) {
7202 trace_pci_nvme_err_startfail_page_too_large(
7203 NVME_CC_MPS(cc),
7204 NVME_CAP_MPSMAX(cap));
7205 return -1;
7206 }
7207 if (unlikely(NVME_CC_IOCQES(cc) <
7208 NVME_CTRL_CQES_MIN(n->id_ctrl.cqes))) {
7209 trace_pci_nvme_err_startfail_cqent_too_small(
7210 NVME_CC_IOCQES(cc),
7211 NVME_CTRL_CQES_MIN(cap));
7212 return -1;
7213 }
7214 if (unlikely(NVME_CC_IOCQES(cc) >
7215 NVME_CTRL_CQES_MAX(n->id_ctrl.cqes))) {
7216 trace_pci_nvme_err_startfail_cqent_too_large(
7217 NVME_CC_IOCQES(cc),
7218 NVME_CTRL_CQES_MAX(cap));
7219 return -1;
7220 }
7221 if (unlikely(NVME_CC_IOSQES(cc) <
7222 NVME_CTRL_SQES_MIN(n->id_ctrl.sqes))) {
7223 trace_pci_nvme_err_startfail_sqent_too_small(
7224 NVME_CC_IOSQES(cc),
7225 NVME_CTRL_SQES_MIN(cap));
7226 return -1;
7227 }
7228 if (unlikely(NVME_CC_IOSQES(cc) >
7229 NVME_CTRL_SQES_MAX(n->id_ctrl.sqes))) {
7230 trace_pci_nvme_err_startfail_sqent_too_large(
7231 NVME_CC_IOSQES(cc),
7232 NVME_CTRL_SQES_MAX(cap));
7233 return -1;
7234 }
7235 if (unlikely(!NVME_AQA_ASQS(aqa))) {
7236 trace_pci_nvme_err_startfail_asqent_sz_zero();
7237 return -1;
7238 }
7239 if (unlikely(!NVME_AQA_ACQS(aqa))) {
7240 trace_pci_nvme_err_startfail_acqent_sz_zero();
7241 return -1;
7242 }
7243
7244 n->page_bits = page_bits;
7245 n->page_size = page_size;
7246 n->max_prp_ents = n->page_size / sizeof(uint64_t);
7247 n->cqe_size = 1 << NVME_CC_IOCQES(cc);
7248 n->sqe_size = 1 << NVME_CC_IOSQES(cc);
7249 nvme_init_cq(&n->admin_cq, n, acq, 0, 0, NVME_AQA_ACQS(aqa) + 1, 1);
7250 nvme_init_sq(&n->admin_sq, n, asq, 0, 0, NVME_AQA_ASQS(aqa) + 1);
7251
7252 nvme_set_timestamp(n, 0ULL);
7253
7254 nvme_select_iocs(n);
7255
7256 return 0;
7257 }
7258
7259 static void nvme_cmb_enable_regs(NvmeCtrl *n)
7260 {
7261 uint32_t cmbloc = ldl_le_p(&n->bar.cmbloc);
7262 uint32_t cmbsz = ldl_le_p(&n->bar.cmbsz);
7263
7264 NVME_CMBLOC_SET_CDPCILS(cmbloc, 1);
7265 NVME_CMBLOC_SET_CDPMLS(cmbloc, 1);
7266 NVME_CMBLOC_SET_BIR(cmbloc, NVME_CMB_BIR);
7267 stl_le_p(&n->bar.cmbloc, cmbloc);
7268
7269 NVME_CMBSZ_SET_SQS(cmbsz, 1);
7270 NVME_CMBSZ_SET_CQS(cmbsz, 0);
7271 NVME_CMBSZ_SET_LISTS(cmbsz, 1);
7272 NVME_CMBSZ_SET_RDS(cmbsz, 1);
7273 NVME_CMBSZ_SET_WDS(cmbsz, 1);
7274 NVME_CMBSZ_SET_SZU(cmbsz, 2); /* MBs */
7275 NVME_CMBSZ_SET_SZ(cmbsz, n->params.cmb_size_mb);
7276 stl_le_p(&n->bar.cmbsz, cmbsz);
7277 }
7278
7279 static void nvme_write_bar(NvmeCtrl *n, hwaddr offset, uint64_t data,
7280 unsigned size)
7281 {
7282 PCIDevice *pci = PCI_DEVICE(n);
7283 uint64_t cap = ldq_le_p(&n->bar.cap);
7284 uint32_t cc = ldl_le_p(&n->bar.cc);
7285 uint32_t intms = ldl_le_p(&n->bar.intms);
7286 uint32_t csts = ldl_le_p(&n->bar.csts);
7287 uint32_t pmrsts = ldl_le_p(&n->bar.pmrsts);
7288
7289 if (unlikely(offset & (sizeof(uint32_t) - 1))) {
7290 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_misaligned32,
7291 "MMIO write not 32-bit aligned,"
7292 " offset=0x%"PRIx64"", offset);
7293 /* should be ignored, fall through for now */
7294 }
7295
7296 if (unlikely(size < sizeof(uint32_t))) {
7297 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_toosmall,
7298 "MMIO write smaller than 32-bits,"
7299 " offset=0x%"PRIx64", size=%u",
7300 offset, size);
7301 /* should be ignored, fall through for now */
7302 }
7303
7304 switch (offset) {
7305 case NVME_REG_INTMS:
7306 if (unlikely(msix_enabled(pci))) {
7307 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix,
7308 "undefined access to interrupt mask set"
7309 " when MSI-X is enabled");
7310 /* should be ignored, fall through for now */
7311 }
7312 intms |= data;
7313 stl_le_p(&n->bar.intms, intms);
7314 n->bar.intmc = n->bar.intms;
7315 trace_pci_nvme_mmio_intm_set(data & 0xffffffff, intms);
7316 nvme_irq_check(n);
7317 break;
7318 case NVME_REG_INTMC:
7319 if (unlikely(msix_enabled(pci))) {
7320 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix,
7321 "undefined access to interrupt mask clr"
7322 " when MSI-X is enabled");
7323 /* should be ignored, fall through for now */
7324 }
7325 intms &= ~data;
7326 stl_le_p(&n->bar.intms, intms);
7327 n->bar.intmc = n->bar.intms;
7328 trace_pci_nvme_mmio_intm_clr(data & 0xffffffff, intms);
7329 nvme_irq_check(n);
7330 break;
7331 case NVME_REG_CC:
7332 stl_le_p(&n->bar.cc, data);
7333
7334 trace_pci_nvme_mmio_cfg(data & 0xffffffff);
7335
7336 if (NVME_CC_SHN(data) && !(NVME_CC_SHN(cc))) {
7337 trace_pci_nvme_mmio_shutdown_set();
7338 nvme_ctrl_shutdown(n);
7339 csts &= ~(CSTS_SHST_MASK << CSTS_SHST_SHIFT);
7340 csts |= NVME_CSTS_SHST_COMPLETE;
7341 } else if (!NVME_CC_SHN(data) && NVME_CC_SHN(cc)) {
7342 trace_pci_nvme_mmio_shutdown_cleared();
7343 csts &= ~(CSTS_SHST_MASK << CSTS_SHST_SHIFT);
7344 }
7345
7346 if (NVME_CC_EN(data) && !NVME_CC_EN(cc)) {
7347 if (unlikely(nvme_start_ctrl(n))) {
7348 trace_pci_nvme_err_startfail();
7349 csts = NVME_CSTS_FAILED;
7350 } else {
7351 trace_pci_nvme_mmio_start_success();
7352 csts = NVME_CSTS_READY;
7353 }
7354 } else if (!NVME_CC_EN(data) && NVME_CC_EN(cc)) {
7355 trace_pci_nvme_mmio_stopped();
7356 nvme_ctrl_reset(n, NVME_RESET_CONTROLLER);
7357
7358 break;
7359 }
7360
7361 stl_le_p(&n->bar.csts, csts);
7362
7363 break;
7364 case NVME_REG_CSTS:
7365 if (data & (1 << 4)) {
7366 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ssreset_w1c_unsupported,
7367 "attempted to W1C CSTS.NSSRO"
7368 " but CAP.NSSRS is zero (not supported)");
7369 } else if (data != 0) {
7370 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ro_csts,
7371 "attempted to set a read only bit"
7372 " of controller status");
7373 }
7374 break;
7375 case NVME_REG_NSSR:
7376 if (data == 0x4e564d65) {
7377 trace_pci_nvme_ub_mmiowr_ssreset_unsupported();
7378 } else {
7379 /* The spec says that writes of other values have no effect */
7380 return;
7381 }
7382 break;
7383 case NVME_REG_AQA:
7384 stl_le_p(&n->bar.aqa, data);
7385 trace_pci_nvme_mmio_aqattr(data & 0xffffffff);
7386 break;
7387 case NVME_REG_ASQ:
7388 stn_le_p(&n->bar.asq, size, data);
7389 trace_pci_nvme_mmio_asqaddr(data);
7390 break;
7391 case NVME_REG_ASQ + 4:
7392 stl_le_p((uint8_t *)&n->bar.asq + 4, data);
7393 trace_pci_nvme_mmio_asqaddr_hi(data, ldq_le_p(&n->bar.asq));
7394 break;
7395 case NVME_REG_ACQ:
7396 trace_pci_nvme_mmio_acqaddr(data);
7397 stn_le_p(&n->bar.acq, size, data);
7398 break;
7399 case NVME_REG_ACQ + 4:
7400 stl_le_p((uint8_t *)&n->bar.acq + 4, data);
7401 trace_pci_nvme_mmio_acqaddr_hi(data, ldq_le_p(&n->bar.acq));
7402 break;
7403 case NVME_REG_CMBLOC:
7404 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbloc_reserved,
7405 "invalid write to reserved CMBLOC"
7406 " when CMBSZ is zero, ignored");
7407 return;
7408 case NVME_REG_CMBSZ:
7409 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbsz_readonly,
7410 "invalid write to read only CMBSZ, ignored");
7411 return;
7412 case NVME_REG_CMBMSC:
7413 if (!NVME_CAP_CMBS(cap)) {
7414 return;
7415 }
7416
7417 stn_le_p(&n->bar.cmbmsc, size, data);
7418 n->cmb.cmse = false;
7419
7420 if (NVME_CMBMSC_CRE(data)) {
7421 nvme_cmb_enable_regs(n);
7422
7423 if (NVME_CMBMSC_CMSE(data)) {
7424 uint64_t cmbmsc = ldq_le_p(&n->bar.cmbmsc);
7425 hwaddr cba = NVME_CMBMSC_CBA(cmbmsc) << CMBMSC_CBA_SHIFT;
7426 if (cba + int128_get64(n->cmb.mem.size) < cba) {
7427 uint32_t cmbsts = ldl_le_p(&n->bar.cmbsts);
7428 NVME_CMBSTS_SET_CBAI(cmbsts, 1);
7429 stl_le_p(&n->bar.cmbsts, cmbsts);
7430 return;
7431 }
7432
7433 n->cmb.cba = cba;
7434 n->cmb.cmse = true;
7435 }
7436 } else {
7437 n->bar.cmbsz = 0;
7438 n->bar.cmbloc = 0;
7439 }
7440
7441 return;
7442 case NVME_REG_CMBMSC + 4:
7443 stl_le_p((uint8_t *)&n->bar.cmbmsc + 4, data);
7444 return;
7445
7446 case NVME_REG_PMRCAP:
7447 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrcap_readonly,
7448 "invalid write to PMRCAP register, ignored");
7449 return;
7450 case NVME_REG_PMRCTL:
7451 if (!NVME_CAP_PMRS(cap)) {
7452 return;
7453 }
7454
7455 stl_le_p(&n->bar.pmrctl, data);
7456 if (NVME_PMRCTL_EN(data)) {
7457 memory_region_set_enabled(&n->pmr.dev->mr, true);
7458 pmrsts = 0;
7459 } else {
7460 memory_region_set_enabled(&n->pmr.dev->mr, false);
7461 NVME_PMRSTS_SET_NRDY(pmrsts, 1);
7462 n->pmr.cmse = false;
7463 }
7464 stl_le_p(&n->bar.pmrsts, pmrsts);
7465 return;
7466 case NVME_REG_PMRSTS:
7467 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrsts_readonly,
7468 "invalid write to PMRSTS register, ignored");
7469 return;
7470 case NVME_REG_PMREBS:
7471 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrebs_readonly,
7472 "invalid write to PMREBS register, ignored");
7473 return;
7474 case NVME_REG_PMRSWTP:
7475 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrswtp_readonly,
7476 "invalid write to PMRSWTP register, ignored");
7477 return;
7478 case NVME_REG_PMRMSCL:
7479 if (!NVME_CAP_PMRS(cap)) {
7480 return;
7481 }
7482
7483 stl_le_p(&n->bar.pmrmscl, data);
7484 n->pmr.cmse = false;
7485
7486 if (NVME_PMRMSCL_CMSE(data)) {
7487 uint64_t pmrmscu = ldl_le_p(&n->bar.pmrmscu);
7488 hwaddr cba = pmrmscu << 32 |
7489 (NVME_PMRMSCL_CBA(data) << PMRMSCL_CBA_SHIFT);
7490 if (cba + int128_get64(n->pmr.dev->mr.size) < cba) {
7491 NVME_PMRSTS_SET_CBAI(pmrsts, 1);
7492 stl_le_p(&n->bar.pmrsts, pmrsts);
7493 return;
7494 }
7495
7496 n->pmr.cmse = true;
7497 n->pmr.cba = cba;
7498 }
7499
7500 return;
7501 case NVME_REG_PMRMSCU:
7502 if (!NVME_CAP_PMRS(cap)) {
7503 return;
7504 }
7505
7506 stl_le_p(&n->bar.pmrmscu, data);
7507 return;
7508 default:
7509 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_invalid,
7510 "invalid MMIO write,"
7511 " offset=0x%"PRIx64", data=%"PRIx64"",
7512 offset, data);
7513 break;
7514 }
7515 }
7516
7517 static uint64_t nvme_mmio_read(void *opaque, hwaddr addr, unsigned size)
7518 {
7519 NvmeCtrl *n = (NvmeCtrl *)opaque;
7520 uint8_t *ptr = (uint8_t *)&n->bar;
7521
7522 trace_pci_nvme_mmio_read(addr, size);
7523
7524 if (unlikely(addr & (sizeof(uint32_t) - 1))) {
7525 NVME_GUEST_ERR(pci_nvme_ub_mmiord_misaligned32,
7526 "MMIO read not 32-bit aligned,"
7527 " offset=0x%"PRIx64"", addr);
7528 /* should RAZ, fall through for now */
7529 } else if (unlikely(size < sizeof(uint32_t))) {
7530 NVME_GUEST_ERR(pci_nvme_ub_mmiord_toosmall,
7531 "MMIO read smaller than 32-bits,"
7532 " offset=0x%"PRIx64"", addr);
7533 /* should RAZ, fall through for now */
7534 }
7535
7536 if (addr > sizeof(n->bar) - size) {
7537 NVME_GUEST_ERR(pci_nvme_ub_mmiord_invalid_ofs,
7538 "MMIO read beyond last register,"
7539 " offset=0x%"PRIx64", returning 0", addr);
7540
7541 return 0;
7542 }
7543
7544 if (pci_is_vf(PCI_DEVICE(n)) && !nvme_sctrl(n)->scs &&
7545 addr != NVME_REG_CSTS) {
7546 trace_pci_nvme_err_ignored_mmio_vf_offline(addr, size);
7547 return 0;
7548 }
7549
7550 /*
7551 * When PMRWBM bit 1 is set then read from
7552 * from PMRSTS should ensure prior writes
7553 * made it to persistent media
7554 */
7555 if (addr == NVME_REG_PMRSTS &&
7556 (NVME_PMRCAP_PMRWBM(ldl_le_p(&n->bar.pmrcap)) & 0x02)) {
7557 memory_region_msync(&n->pmr.dev->mr, 0, n->pmr.dev->size);
7558 }
7559
7560 return ldn_le_p(ptr + addr, size);
7561 }
7562
7563 static void nvme_process_db(NvmeCtrl *n, hwaddr addr, int val)
7564 {
7565 PCIDevice *pci = PCI_DEVICE(n);
7566 uint32_t qid;
7567
7568 if (unlikely(addr & ((1 << 2) - 1))) {
7569 NVME_GUEST_ERR(pci_nvme_ub_db_wr_misaligned,
7570 "doorbell write not 32-bit aligned,"
7571 " offset=0x%"PRIx64", ignoring", addr);
7572 return;
7573 }
7574
7575 if (((addr - 0x1000) >> 2) & 1) {
7576 /* Completion queue doorbell write */
7577
7578 uint16_t new_head = val & 0xffff;
7579 int start_sqs;
7580 NvmeCQueue *cq;
7581
7582 qid = (addr - (0x1000 + (1 << 2))) >> 3;
7583 if (unlikely(nvme_check_cqid(n, qid))) {
7584 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cq,
7585 "completion queue doorbell write"
7586 " for nonexistent queue,"
7587 " sqid=%"PRIu32", ignoring", qid);
7588
7589 /*
7590 * NVM Express v1.3d, Section 4.1 state: "If host software writes
7591 * an invalid value to the Submission Queue Tail Doorbell or
7592 * Completion Queue Head Doorbell regiter and an Asynchronous Event
7593 * Request command is outstanding, then an asynchronous event is
7594 * posted to the Admin Completion Queue with a status code of
7595 * Invalid Doorbell Write Value."
7596 *
7597 * Also note that the spec includes the "Invalid Doorbell Register"
7598 * status code, but nowhere does it specify when to use it.
7599 * However, it seems reasonable to use it here in a similar
7600 * fashion.
7601 */
7602 if (n->outstanding_aers) {
7603 nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
7604 NVME_AER_INFO_ERR_INVALID_DB_REGISTER,
7605 NVME_LOG_ERROR_INFO);
7606 }
7607
7608 return;
7609 }
7610
7611 cq = n->cq[qid];
7612 if (unlikely(new_head >= cq->size)) {
7613 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cqhead,
7614 "completion queue doorbell write value"
7615 " beyond queue size, sqid=%"PRIu32","
7616 " new_head=%"PRIu16", ignoring",
7617 qid, new_head);
7618
7619 if (n->outstanding_aers) {
7620 nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
7621 NVME_AER_INFO_ERR_INVALID_DB_VALUE,
7622 NVME_LOG_ERROR_INFO);
7623 }
7624
7625 return;
7626 }
7627
7628 trace_pci_nvme_mmio_doorbell_cq(cq->cqid, new_head);
7629
7630 start_sqs = nvme_cq_full(cq) ? 1 : 0;
7631 cq->head = new_head;
7632 if (!qid && n->dbbuf_enabled) {
7633 pci_dma_write(pci, cq->db_addr, &cq->head, sizeof(cq->head));
7634 }
7635 if (start_sqs) {
7636 NvmeSQueue *sq;
7637 QTAILQ_FOREACH(sq, &cq->sq_list, entry) {
7638 qemu_bh_schedule(sq->bh);
7639 }
7640 qemu_bh_schedule(cq->bh);
7641 }
7642
7643 if (cq->tail == cq->head) {
7644 if (cq->irq_enabled) {
7645 n->cq_pending--;
7646 }
7647
7648 nvme_irq_deassert(n, cq);
7649 }
7650 } else {
7651 /* Submission queue doorbell write */
7652
7653 uint16_t new_tail = val & 0xffff;
7654 NvmeSQueue *sq;
7655
7656 qid = (addr - 0x1000) >> 3;
7657 if (unlikely(nvme_check_sqid(n, qid))) {
7658 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sq,
7659 "submission queue doorbell write"
7660 " for nonexistent queue,"
7661 " sqid=%"PRIu32", ignoring", qid);
7662
7663 if (n->outstanding_aers) {
7664 nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
7665 NVME_AER_INFO_ERR_INVALID_DB_REGISTER,
7666 NVME_LOG_ERROR_INFO);
7667 }
7668
7669 return;
7670 }
7671
7672 sq = n->sq[qid];
7673 if (unlikely(new_tail >= sq->size)) {
7674 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sqtail,
7675 "submission queue doorbell write value"
7676 " beyond queue size, sqid=%"PRIu32","
7677 " new_tail=%"PRIu16", ignoring",
7678 qid, new_tail);
7679
7680 if (n->outstanding_aers) {
7681 nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
7682 NVME_AER_INFO_ERR_INVALID_DB_VALUE,
7683 NVME_LOG_ERROR_INFO);
7684 }
7685
7686 return;
7687 }
7688
7689 trace_pci_nvme_mmio_doorbell_sq(sq->sqid, new_tail);
7690
7691 sq->tail = new_tail;
7692 if (!qid && n->dbbuf_enabled) {
7693 /*
7694 * The spec states "the host shall also update the controller's
7695 * corresponding doorbell property to match the value of that entry
7696 * in the Shadow Doorbell buffer."
7697 *
7698 * Since this context is currently a VM trap, we can safely enforce
7699 * the requirement from the device side in case the host is
7700 * misbehaving.
7701 *
7702 * Note, we shouldn't have to do this, but various drivers
7703 * including ones that run on Linux, are not updating Admin Queues,
7704 * so we can't trust reading it for an appropriate sq tail.
7705 */
7706 pci_dma_write(pci, sq->db_addr, &sq->tail, sizeof(sq->tail));
7707 }
7708
7709 qemu_bh_schedule(sq->bh);
7710 }
7711 }
7712
7713 static void nvme_mmio_write(void *opaque, hwaddr addr, uint64_t data,
7714 unsigned size)
7715 {
7716 NvmeCtrl *n = (NvmeCtrl *)opaque;
7717
7718 trace_pci_nvme_mmio_write(addr, data, size);
7719
7720 if (pci_is_vf(PCI_DEVICE(n)) && !nvme_sctrl(n)->scs &&
7721 addr != NVME_REG_CSTS) {
7722 trace_pci_nvme_err_ignored_mmio_vf_offline(addr, size);
7723 return;
7724 }
7725
7726 if (addr < sizeof(n->bar)) {
7727 nvme_write_bar(n, addr, data, size);
7728 } else {
7729 nvme_process_db(n, addr, data);
7730 }
7731 }
7732
7733 static const MemoryRegionOps nvme_mmio_ops = {
7734 .read = nvme_mmio_read,
7735 .write = nvme_mmio_write,
7736 .endianness = DEVICE_LITTLE_ENDIAN,
7737 .impl = {
7738 .min_access_size = 2,
7739 .max_access_size = 8,
7740 },
7741 };
7742
7743 static void nvme_cmb_write(void *opaque, hwaddr addr, uint64_t data,
7744 unsigned size)
7745 {
7746 NvmeCtrl *n = (NvmeCtrl *)opaque;
7747 stn_le_p(&n->cmb.buf[addr], size, data);
7748 }
7749
7750 static uint64_t nvme_cmb_read(void *opaque, hwaddr addr, unsigned size)
7751 {
7752 NvmeCtrl *n = (NvmeCtrl *)opaque;
7753 return ldn_le_p(&n->cmb.buf[addr], size);
7754 }
7755
7756 static const MemoryRegionOps nvme_cmb_ops = {
7757 .read = nvme_cmb_read,
7758 .write = nvme_cmb_write,
7759 .endianness = DEVICE_LITTLE_ENDIAN,
7760 .impl = {
7761 .min_access_size = 1,
7762 .max_access_size = 8,
7763 },
7764 };
7765
7766 static bool nvme_check_params(NvmeCtrl *n, Error **errp)
7767 {
7768 NvmeParams *params = &n->params;
7769
7770 if (params->num_queues) {
7771 warn_report("num_queues is deprecated; please use max_ioqpairs "
7772 "instead");
7773
7774 params->max_ioqpairs = params->num_queues - 1;
7775 }
7776
7777 if (n->namespace.blkconf.blk && n->subsys) {
7778 error_setg(errp, "subsystem support is unavailable with legacy "
7779 "namespace ('drive' property)");
7780 return false;
7781 }
7782
7783 if (params->max_ioqpairs < 1 ||
7784 params->max_ioqpairs > NVME_MAX_IOQPAIRS) {
7785 error_setg(errp, "max_ioqpairs must be between 1 and %d",
7786 NVME_MAX_IOQPAIRS);
7787 return false;
7788 }
7789
7790 if (params->msix_qsize < 1 ||
7791 params->msix_qsize > PCI_MSIX_FLAGS_QSIZE + 1) {
7792 error_setg(errp, "msix_qsize must be between 1 and %d",
7793 PCI_MSIX_FLAGS_QSIZE + 1);
7794 return false;
7795 }
7796
7797 if (!params->serial) {
7798 error_setg(errp, "serial property not set");
7799 return false;
7800 }
7801
7802 if (n->pmr.dev) {
7803 if (host_memory_backend_is_mapped(n->pmr.dev)) {
7804 error_setg(errp, "can't use already busy memdev: %s",
7805 object_get_canonical_path_component(OBJECT(n->pmr.dev)));
7806 return false;
7807 }
7808
7809 if (!is_power_of_2(n->pmr.dev->size)) {
7810 error_setg(errp, "pmr backend size needs to be power of 2 in size");
7811 return false;
7812 }
7813
7814 host_memory_backend_set_mapped(n->pmr.dev, true);
7815 }
7816
7817 if (n->params.zasl > n->params.mdts) {
7818 error_setg(errp, "zoned.zasl (Zone Append Size Limit) must be less "
7819 "than or equal to mdts (Maximum Data Transfer Size)");
7820 return false;
7821 }
7822
7823 if (!n->params.vsl) {
7824 error_setg(errp, "vsl must be non-zero");
7825 return false;
7826 }
7827
7828 if (params->sriov_max_vfs) {
7829 if (!n->subsys) {
7830 error_setg(errp, "subsystem is required for the use of SR-IOV");
7831 return false;
7832 }
7833
7834 if (params->sriov_max_vfs > NVME_MAX_VFS) {
7835 error_setg(errp, "sriov_max_vfs must be between 0 and %d",
7836 NVME_MAX_VFS);
7837 return false;
7838 }
7839
7840 if (params->cmb_size_mb) {
7841 error_setg(errp, "CMB is not supported with SR-IOV");
7842 return false;
7843 }
7844
7845 if (n->pmr.dev) {
7846 error_setg(errp, "PMR is not supported with SR-IOV");
7847 return false;
7848 }
7849
7850 if (!params->sriov_vq_flexible || !params->sriov_vi_flexible) {
7851 error_setg(errp, "both sriov_vq_flexible and sriov_vi_flexible"
7852 " must be set for the use of SR-IOV");
7853 return false;
7854 }
7855
7856 if (params->sriov_vq_flexible < params->sriov_max_vfs * 2) {
7857 error_setg(errp, "sriov_vq_flexible must be greater than or equal"
7858 " to %d (sriov_max_vfs * 2)", params->sriov_max_vfs * 2);
7859 return false;
7860 }
7861
7862 if (params->max_ioqpairs < params->sriov_vq_flexible + 2) {
7863 error_setg(errp, "(max_ioqpairs - sriov_vq_flexible) must be"
7864 " greater than or equal to 2");
7865 return false;
7866 }
7867
7868 if (params->sriov_vi_flexible < params->sriov_max_vfs) {
7869 error_setg(errp, "sriov_vi_flexible must be greater than or equal"
7870 " to %d (sriov_max_vfs)", params->sriov_max_vfs);
7871 return false;
7872 }
7873
7874 if (params->msix_qsize < params->sriov_vi_flexible + 1) {
7875 error_setg(errp, "(msix_qsize - sriov_vi_flexible) must be"
7876 " greater than or equal to 1");
7877 return false;
7878 }
7879
7880 if (params->sriov_max_vi_per_vf &&
7881 (params->sriov_max_vi_per_vf - 1) % NVME_VF_RES_GRANULARITY) {
7882 error_setg(errp, "sriov_max_vi_per_vf must meet:"
7883 " (sriov_max_vi_per_vf - 1) %% %d == 0 and"
7884 " sriov_max_vi_per_vf >= 1", NVME_VF_RES_GRANULARITY);
7885 return false;
7886 }
7887
7888 if (params->sriov_max_vq_per_vf &&
7889 (params->sriov_max_vq_per_vf < 2 ||
7890 (params->sriov_max_vq_per_vf - 1) % NVME_VF_RES_GRANULARITY)) {
7891 error_setg(errp, "sriov_max_vq_per_vf must meet:"
7892 " (sriov_max_vq_per_vf - 1) %% %d == 0 and"
7893 " sriov_max_vq_per_vf >= 2", NVME_VF_RES_GRANULARITY);
7894 return false;
7895 }
7896 }
7897
7898 return true;
7899 }
7900
7901 static void nvme_init_state(NvmeCtrl *n)
7902 {
7903 NvmePriCtrlCap *cap = &n->pri_ctrl_cap;
7904 NvmeSecCtrlList *list = &n->sec_ctrl_list;
7905 NvmeSecCtrlEntry *sctrl;
7906 PCIDevice *pci = PCI_DEVICE(n);
7907 uint8_t max_vfs;
7908 int i;
7909
7910 if (pci_is_vf(pci)) {
7911 sctrl = nvme_sctrl(n);
7912 max_vfs = 0;
7913 n->conf_ioqpairs = sctrl->nvq ? le16_to_cpu(sctrl->nvq) - 1 : 0;
7914 n->conf_msix_qsize = sctrl->nvi ? le16_to_cpu(sctrl->nvi) : 1;
7915 } else {
7916 max_vfs = n->params.sriov_max_vfs;
7917 n->conf_ioqpairs = n->params.max_ioqpairs;
7918 n->conf_msix_qsize = n->params.msix_qsize;
7919 }
7920
7921 n->sq = g_new0(NvmeSQueue *, n->params.max_ioqpairs + 1);
7922 n->cq = g_new0(NvmeCQueue *, n->params.max_ioqpairs + 1);
7923 n->temperature = NVME_TEMPERATURE;
7924 n->features.temp_thresh_hi = NVME_TEMPERATURE_WARNING;
7925 n->starttime_ms = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
7926 n->aer_reqs = g_new0(NvmeRequest *, n->params.aerl + 1);
7927 QTAILQ_INIT(&n->aer_queue);
7928
7929 list->numcntl = cpu_to_le16(max_vfs);
7930 for (i = 0; i < max_vfs; i++) {
7931 sctrl = &list->sec[i];
7932 sctrl->pcid = cpu_to_le16(n->cntlid);
7933 sctrl->vfn = cpu_to_le16(i + 1);
7934 }
7935
7936 cap->cntlid = cpu_to_le16(n->cntlid);
7937 cap->crt = NVME_CRT_VQ | NVME_CRT_VI;
7938
7939 if (pci_is_vf(pci)) {
7940 cap->vqprt = cpu_to_le16(1 + n->conf_ioqpairs);
7941 } else {
7942 cap->vqprt = cpu_to_le16(1 + n->params.max_ioqpairs -
7943 n->params.sriov_vq_flexible);
7944 cap->vqfrt = cpu_to_le32(n->params.sriov_vq_flexible);
7945 cap->vqrfap = cap->vqfrt;
7946 cap->vqgran = cpu_to_le16(NVME_VF_RES_GRANULARITY);
7947 cap->vqfrsm = n->params.sriov_max_vq_per_vf ?
7948 cpu_to_le16(n->params.sriov_max_vq_per_vf) :
7949 cap->vqfrt / MAX(max_vfs, 1);
7950 }
7951
7952 if (pci_is_vf(pci)) {
7953 cap->viprt = cpu_to_le16(n->conf_msix_qsize);
7954 } else {
7955 cap->viprt = cpu_to_le16(n->params.msix_qsize -
7956 n->params.sriov_vi_flexible);
7957 cap->vifrt = cpu_to_le32(n->params.sriov_vi_flexible);
7958 cap->virfap = cap->vifrt;
7959 cap->vigran = cpu_to_le16(NVME_VF_RES_GRANULARITY);
7960 cap->vifrsm = n->params.sriov_max_vi_per_vf ?
7961 cpu_to_le16(n->params.sriov_max_vi_per_vf) :
7962 cap->vifrt / MAX(max_vfs, 1);
7963 }
7964 }
7965
7966 static void nvme_init_cmb(NvmeCtrl *n, PCIDevice *pci_dev)
7967 {
7968 uint64_t cmb_size = n->params.cmb_size_mb * MiB;
7969 uint64_t cap = ldq_le_p(&n->bar.cap);
7970
7971 n->cmb.buf = g_malloc0(cmb_size);
7972 memory_region_init_io(&n->cmb.mem, OBJECT(n), &nvme_cmb_ops, n,
7973 "nvme-cmb", cmb_size);
7974 pci_register_bar(pci_dev, NVME_CMB_BIR,
7975 PCI_BASE_ADDRESS_SPACE_MEMORY |
7976 PCI_BASE_ADDRESS_MEM_TYPE_64 |
7977 PCI_BASE_ADDRESS_MEM_PREFETCH, &n->cmb.mem);
7978
7979 NVME_CAP_SET_CMBS(cap, 1);
7980 stq_le_p(&n->bar.cap, cap);
7981
7982 if (n->params.legacy_cmb) {
7983 nvme_cmb_enable_regs(n);
7984 n->cmb.cmse = true;
7985 }
7986 }
7987
7988 static void nvme_init_pmr(NvmeCtrl *n, PCIDevice *pci_dev)
7989 {
7990 uint32_t pmrcap = ldl_le_p(&n->bar.pmrcap);
7991
7992 NVME_PMRCAP_SET_RDS(pmrcap, 1);
7993 NVME_PMRCAP_SET_WDS(pmrcap, 1);
7994 NVME_PMRCAP_SET_BIR(pmrcap, NVME_PMR_BIR);
7995 /* Turn on bit 1 support */
7996 NVME_PMRCAP_SET_PMRWBM(pmrcap, 0x02);
7997 NVME_PMRCAP_SET_CMSS(pmrcap, 1);
7998 stl_le_p(&n->bar.pmrcap, pmrcap);
7999
8000 pci_register_bar(pci_dev, NVME_PMR_BIR,
8001 PCI_BASE_ADDRESS_SPACE_MEMORY |
8002 PCI_BASE_ADDRESS_MEM_TYPE_64 |
8003 PCI_BASE_ADDRESS_MEM_PREFETCH, &n->pmr.dev->mr);
8004
8005 memory_region_set_enabled(&n->pmr.dev->mr, false);
8006 }
8007
8008 static uint64_t nvme_bar_size(unsigned total_queues, unsigned total_irqs,
8009 unsigned *msix_table_offset,
8010 unsigned *msix_pba_offset)
8011 {
8012 uint64_t bar_size, msix_table_size, msix_pba_size;
8013
8014 bar_size = sizeof(NvmeBar) + 2 * total_queues * NVME_DB_SIZE;
8015 bar_size = QEMU_ALIGN_UP(bar_size, 4 * KiB);
8016
8017 if (msix_table_offset) {
8018 *msix_table_offset = bar_size;
8019 }
8020
8021 msix_table_size = PCI_MSIX_ENTRY_SIZE * total_irqs;
8022 bar_size += msix_table_size;
8023 bar_size = QEMU_ALIGN_UP(bar_size, 4 * KiB);
8024
8025 if (msix_pba_offset) {
8026 *msix_pba_offset = bar_size;
8027 }
8028
8029 msix_pba_size = QEMU_ALIGN_UP(total_irqs, 64) / 8;
8030 bar_size += msix_pba_size;
8031
8032 bar_size = pow2ceil(bar_size);
8033 return bar_size;
8034 }
8035
8036 static void nvme_init_sriov(NvmeCtrl *n, PCIDevice *pci_dev, uint16_t offset)
8037 {
8038 uint16_t vf_dev_id = n->params.use_intel_id ?
8039 PCI_DEVICE_ID_INTEL_NVME : PCI_DEVICE_ID_REDHAT_NVME;
8040 NvmePriCtrlCap *cap = &n->pri_ctrl_cap;
8041 uint64_t bar_size = nvme_bar_size(le16_to_cpu(cap->vqfrsm),
8042 le16_to_cpu(cap->vifrsm),
8043 NULL, NULL);
8044
8045 pcie_sriov_pf_init(pci_dev, offset, "nvme", vf_dev_id,
8046 n->params.sriov_max_vfs, n->params.sriov_max_vfs,
8047 NVME_VF_OFFSET, NVME_VF_STRIDE);
8048
8049 pcie_sriov_pf_init_vf_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY |
8050 PCI_BASE_ADDRESS_MEM_TYPE_64, bar_size);
8051 }
8052
8053 static int nvme_add_pm_capability(PCIDevice *pci_dev, uint8_t offset)
8054 {
8055 Error *err = NULL;
8056 int ret;
8057
8058 ret = pci_add_capability(pci_dev, PCI_CAP_ID_PM, offset,
8059 PCI_PM_SIZEOF, &err);
8060 if (err) {
8061 error_report_err(err);
8062 return ret;
8063 }
8064
8065 pci_set_word(pci_dev->config + offset + PCI_PM_PMC,
8066 PCI_PM_CAP_VER_1_2);
8067 pci_set_word(pci_dev->config + offset + PCI_PM_CTRL,
8068 PCI_PM_CTRL_NO_SOFT_RESET);
8069 pci_set_word(pci_dev->wmask + offset + PCI_PM_CTRL,
8070 PCI_PM_CTRL_STATE_MASK);
8071
8072 return 0;
8073 }
8074
8075 static bool nvme_init_pci(NvmeCtrl *n, PCIDevice *pci_dev, Error **errp)
8076 {
8077 ERRP_GUARD();
8078 uint8_t *pci_conf = pci_dev->config;
8079 uint64_t bar_size;
8080 unsigned msix_table_offset, msix_pba_offset;
8081 int ret;
8082
8083 pci_conf[PCI_INTERRUPT_PIN] = 1;
8084 pci_config_set_prog_interface(pci_conf, 0x2);
8085
8086 if (n->params.use_intel_id) {
8087 pci_config_set_vendor_id(pci_conf, PCI_VENDOR_ID_INTEL);
8088 pci_config_set_device_id(pci_conf, PCI_DEVICE_ID_INTEL_NVME);
8089 } else {
8090 pci_config_set_vendor_id(pci_conf, PCI_VENDOR_ID_REDHAT);
8091 pci_config_set_device_id(pci_conf, PCI_DEVICE_ID_REDHAT_NVME);
8092 }
8093
8094 pci_config_set_class(pci_conf, PCI_CLASS_STORAGE_EXPRESS);
8095 nvme_add_pm_capability(pci_dev, 0x60);
8096 pcie_endpoint_cap_init(pci_dev, 0x80);
8097 pcie_cap_flr_init(pci_dev);
8098 if (n->params.sriov_max_vfs) {
8099 pcie_ari_init(pci_dev, 0x100, 1);
8100 }
8101
8102 /* add one to max_ioqpairs to account for the admin queue pair */
8103 bar_size = nvme_bar_size(n->params.max_ioqpairs + 1, n->params.msix_qsize,
8104 &msix_table_offset, &msix_pba_offset);
8105
8106 memory_region_init(&n->bar0, OBJECT(n), "nvme-bar0", bar_size);
8107 memory_region_init_io(&n->iomem, OBJECT(n), &nvme_mmio_ops, n, "nvme",
8108 msix_table_offset);
8109 memory_region_add_subregion(&n->bar0, 0, &n->iomem);
8110
8111 if (pci_is_vf(pci_dev)) {
8112 pcie_sriov_vf_register_bar(pci_dev, 0, &n->bar0);
8113 } else {
8114 pci_register_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY |
8115 PCI_BASE_ADDRESS_MEM_TYPE_64, &n->bar0);
8116 }
8117 ret = msix_init(pci_dev, n->params.msix_qsize,
8118 &n->bar0, 0, msix_table_offset,
8119 &n->bar0, 0, msix_pba_offset, 0, errp);
8120 if (ret == -ENOTSUP) {
8121 /* report that msix is not supported, but do not error out */
8122 warn_report_err(*errp);
8123 *errp = NULL;
8124 } else if (ret < 0) {
8125 /* propagate error to caller */
8126 return false;
8127 }
8128
8129 nvme_update_msixcap_ts(pci_dev, n->conf_msix_qsize);
8130
8131 if (n->params.cmb_size_mb) {
8132 nvme_init_cmb(n, pci_dev);
8133 }
8134
8135 if (n->pmr.dev) {
8136 nvme_init_pmr(n, pci_dev);
8137 }
8138
8139 if (!pci_is_vf(pci_dev) && n->params.sriov_max_vfs) {
8140 nvme_init_sriov(n, pci_dev, 0x120);
8141 }
8142
8143 return true;
8144 }
8145
8146 static void nvme_init_subnqn(NvmeCtrl *n)
8147 {
8148 NvmeSubsystem *subsys = n->subsys;
8149 NvmeIdCtrl *id = &n->id_ctrl;
8150
8151 if (!subsys) {
8152 snprintf((char *)id->subnqn, sizeof(id->subnqn),
8153 "nqn.2019-08.org.qemu:%s", n->params.serial);
8154 } else {
8155 pstrcpy((char *)id->subnqn, sizeof(id->subnqn), (char*)subsys->subnqn);
8156 }
8157 }
8158
8159 static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev)
8160 {
8161 NvmeIdCtrl *id = &n->id_ctrl;
8162 uint8_t *pci_conf = pci_dev->config;
8163 uint64_t cap = ldq_le_p(&n->bar.cap);
8164 NvmeSecCtrlEntry *sctrl = nvme_sctrl(n);
8165 uint32_t ctratt;
8166
8167 id->vid = cpu_to_le16(pci_get_word(pci_conf + PCI_VENDOR_ID));
8168 id->ssvid = cpu_to_le16(pci_get_word(pci_conf + PCI_SUBSYSTEM_VENDOR_ID));
8169 strpadcpy((char *)id->mn, sizeof(id->mn), "QEMU NVMe Ctrl", ' ');
8170 strpadcpy((char *)id->fr, sizeof(id->fr), QEMU_VERSION, ' ');
8171 strpadcpy((char *)id->sn, sizeof(id->sn), n->params.serial, ' ');
8172
8173 id->cntlid = cpu_to_le16(n->cntlid);
8174
8175 id->oaes = cpu_to_le32(NVME_OAES_NS_ATTR);
8176 ctratt = NVME_CTRATT_ELBAS;
8177
8178 id->rab = 6;
8179
8180 if (n->params.use_intel_id) {
8181 id->ieee[0] = 0xb3;
8182 id->ieee[1] = 0x02;
8183 id->ieee[2] = 0x00;
8184 } else {
8185 id->ieee[0] = 0x00;
8186 id->ieee[1] = 0x54;
8187 id->ieee[2] = 0x52;
8188 }
8189
8190 id->mdts = n->params.mdts;
8191 id->ver = cpu_to_le32(NVME_SPEC_VER);
8192 id->oacs =
8193 cpu_to_le16(NVME_OACS_NS_MGMT | NVME_OACS_FORMAT | NVME_OACS_DBBUF |
8194 NVME_OACS_DIRECTIVES);
8195 id->cntrltype = 0x1;
8196
8197 /*
8198 * Because the controller always completes the Abort command immediately,
8199 * there can never be more than one concurrently executing Abort command,
8200 * so this value is never used for anything. Note that there can easily be
8201 * many Abort commands in the queues, but they are not considered
8202 * "executing" until processed by nvme_abort.
8203 *
8204 * The specification recommends a value of 3 for Abort Command Limit (four
8205 * concurrently outstanding Abort commands), so lets use that though it is
8206 * inconsequential.
8207 */
8208 id->acl = 3;
8209 id->aerl = n->params.aerl;
8210 id->frmw = (NVME_NUM_FW_SLOTS << 1) | NVME_FRMW_SLOT1_RO;
8211 id->lpa = NVME_LPA_NS_SMART | NVME_LPA_CSE | NVME_LPA_EXTENDED;
8212
8213 /* recommended default value (~70 C) */
8214 id->wctemp = cpu_to_le16(NVME_TEMPERATURE_WARNING);
8215 id->cctemp = cpu_to_le16(NVME_TEMPERATURE_CRITICAL);
8216
8217 id->sqes = (0x6 << 4) | 0x6;
8218 id->cqes = (0x4 << 4) | 0x4;
8219 id->nn = cpu_to_le32(NVME_MAX_NAMESPACES);
8220 id->oncs = cpu_to_le16(NVME_ONCS_WRITE_ZEROES | NVME_ONCS_TIMESTAMP |
8221 NVME_ONCS_FEATURES | NVME_ONCS_DSM |
8222 NVME_ONCS_COMPARE | NVME_ONCS_COPY);
8223
8224 /*
8225 * NOTE: If this device ever supports a command set that does NOT use 0x0
8226 * as a Flush-equivalent operation, support for the broadcast NSID in Flush
8227 * should probably be removed.
8228 *
8229 * See comment in nvme_io_cmd.
8230 */
8231 id->vwc = NVME_VWC_NSID_BROADCAST_SUPPORT | NVME_VWC_PRESENT;
8232
8233 id->ocfs = cpu_to_le16(NVME_OCFS_COPY_FORMAT_0 | NVME_OCFS_COPY_FORMAT_1);
8234 id->sgls = cpu_to_le32(NVME_CTRL_SGLS_SUPPORT_NO_ALIGN);
8235
8236 nvme_init_subnqn(n);
8237
8238 id->psd[0].mp = cpu_to_le16(0x9c4);
8239 id->psd[0].enlat = cpu_to_le32(0x10);
8240 id->psd[0].exlat = cpu_to_le32(0x4);
8241
8242 if (n->subsys) {
8243 id->cmic |= NVME_CMIC_MULTI_CTRL;
8244 ctratt |= NVME_CTRATT_ENDGRPS;
8245
8246 id->endgidmax = cpu_to_le16(0x1);
8247
8248 if (n->subsys->endgrp.fdp.enabled) {
8249 ctratt |= NVME_CTRATT_FDPS;
8250 }
8251 }
8252
8253 id->ctratt = cpu_to_le32(ctratt);
8254
8255 NVME_CAP_SET_MQES(cap, 0x7ff);
8256 NVME_CAP_SET_CQR(cap, 1);
8257 NVME_CAP_SET_TO(cap, 0xf);
8258 NVME_CAP_SET_CSS(cap, NVME_CAP_CSS_NVM);
8259 NVME_CAP_SET_CSS(cap, NVME_CAP_CSS_CSI_SUPP);
8260 NVME_CAP_SET_CSS(cap, NVME_CAP_CSS_ADMIN_ONLY);
8261 NVME_CAP_SET_MPSMAX(cap, 4);
8262 NVME_CAP_SET_CMBS(cap, n->params.cmb_size_mb ? 1 : 0);
8263 NVME_CAP_SET_PMRS(cap, n->pmr.dev ? 1 : 0);
8264 stq_le_p(&n->bar.cap, cap);
8265
8266 stl_le_p(&n->bar.vs, NVME_SPEC_VER);
8267 n->bar.intmc = n->bar.intms = 0;
8268
8269 if (pci_is_vf(pci_dev) && !sctrl->scs) {
8270 stl_le_p(&n->bar.csts, NVME_CSTS_FAILED);
8271 }
8272 }
8273
8274 static int nvme_init_subsys(NvmeCtrl *n, Error **errp)
8275 {
8276 int cntlid;
8277
8278 if (!n->subsys) {
8279 return 0;
8280 }
8281
8282 cntlid = nvme_subsys_register_ctrl(n, errp);
8283 if (cntlid < 0) {
8284 return -1;
8285 }
8286
8287 n->cntlid = cntlid;
8288
8289 return 0;
8290 }
8291
8292 void nvme_attach_ns(NvmeCtrl *n, NvmeNamespace *ns)
8293 {
8294 uint32_t nsid = ns->params.nsid;
8295 assert(nsid && nsid <= NVME_MAX_NAMESPACES);
8296
8297 n->namespaces[nsid] = ns;
8298 ns->attached++;
8299
8300 n->dmrsl = MIN_NON_ZERO(n->dmrsl,
8301 BDRV_REQUEST_MAX_BYTES / nvme_l2b(ns, 1));
8302 }
8303
8304 static void nvme_realize(PCIDevice *pci_dev, Error **errp)
8305 {
8306 NvmeCtrl *n = NVME(pci_dev);
8307 DeviceState *dev = DEVICE(pci_dev);
8308 NvmeNamespace *ns;
8309 NvmeCtrl *pn = NVME(pcie_sriov_get_pf(pci_dev));
8310
8311 if (pci_is_vf(pci_dev)) {
8312 /*
8313 * VFs derive settings from the parent. PF's lifespan exceeds
8314 * that of VF's, so it's safe to share params.serial.
8315 */
8316 memcpy(&n->params, &pn->params, sizeof(NvmeParams));
8317 n->subsys = pn->subsys;
8318 }
8319
8320 if (!nvme_check_params(n, errp)) {
8321 return;
8322 }
8323
8324 qbus_init(&n->bus, sizeof(NvmeBus), TYPE_NVME_BUS, dev, dev->id);
8325
8326 if (nvme_init_subsys(n, errp)) {
8327 return;
8328 }
8329 nvme_init_state(n);
8330 if (!nvme_init_pci(n, pci_dev, errp)) {
8331 return;
8332 }
8333 nvme_init_ctrl(n, pci_dev);
8334
8335 /* setup a namespace if the controller drive property was given */
8336 if (n->namespace.blkconf.blk) {
8337 ns = &n->namespace;
8338 ns->params.nsid = 1;
8339
8340 if (nvme_ns_setup(ns, errp)) {
8341 return;
8342 }
8343
8344 nvme_attach_ns(n, ns);
8345 }
8346 }
8347
8348 static void nvme_exit(PCIDevice *pci_dev)
8349 {
8350 NvmeCtrl *n = NVME(pci_dev);
8351 NvmeNamespace *ns;
8352 int i;
8353
8354 nvme_ctrl_reset(n, NVME_RESET_FUNCTION);
8355
8356 if (n->subsys) {
8357 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
8358 ns = nvme_ns(n, i);
8359 if (ns) {
8360 ns->attached--;
8361 }
8362 }
8363
8364 nvme_subsys_unregister_ctrl(n->subsys, n);
8365 }
8366
8367 g_free(n->cq);
8368 g_free(n->sq);
8369 g_free(n->aer_reqs);
8370
8371 if (n->params.cmb_size_mb) {
8372 g_free(n->cmb.buf);
8373 }
8374
8375 if (n->pmr.dev) {
8376 host_memory_backend_set_mapped(n->pmr.dev, false);
8377 }
8378
8379 if (!pci_is_vf(pci_dev) && n->params.sriov_max_vfs) {
8380 pcie_sriov_pf_exit(pci_dev);
8381 }
8382
8383 msix_uninit(pci_dev, &n->bar0, &n->bar0);
8384 memory_region_del_subregion(&n->bar0, &n->iomem);
8385 }
8386
8387 static Property nvme_props[] = {
8388 DEFINE_BLOCK_PROPERTIES(NvmeCtrl, namespace.blkconf),
8389 DEFINE_PROP_LINK("pmrdev", NvmeCtrl, pmr.dev, TYPE_MEMORY_BACKEND,
8390 HostMemoryBackend *),
8391 DEFINE_PROP_LINK("subsys", NvmeCtrl, subsys, TYPE_NVME_SUBSYS,
8392 NvmeSubsystem *),
8393 DEFINE_PROP_STRING("serial", NvmeCtrl, params.serial),
8394 DEFINE_PROP_UINT32("cmb_size_mb", NvmeCtrl, params.cmb_size_mb, 0),
8395 DEFINE_PROP_UINT32("num_queues", NvmeCtrl, params.num_queues, 0),
8396 DEFINE_PROP_UINT32("max_ioqpairs", NvmeCtrl, params.max_ioqpairs, 64),
8397 DEFINE_PROP_UINT16("msix_qsize", NvmeCtrl, params.msix_qsize, 65),
8398 DEFINE_PROP_UINT8("aerl", NvmeCtrl, params.aerl, 3),
8399 DEFINE_PROP_UINT32("aer_max_queued", NvmeCtrl, params.aer_max_queued, 64),
8400 DEFINE_PROP_UINT8("mdts", NvmeCtrl, params.mdts, 7),
8401 DEFINE_PROP_UINT8("vsl", NvmeCtrl, params.vsl, 7),
8402 DEFINE_PROP_BOOL("use-intel-id", NvmeCtrl, params.use_intel_id, false),
8403 DEFINE_PROP_BOOL("legacy-cmb", NvmeCtrl, params.legacy_cmb, false),
8404 DEFINE_PROP_BOOL("ioeventfd", NvmeCtrl, params.ioeventfd, false),
8405 DEFINE_PROP_UINT8("zoned.zasl", NvmeCtrl, params.zasl, 0),
8406 DEFINE_PROP_BOOL("zoned.auto_transition", NvmeCtrl,
8407 params.auto_transition_zones, true),
8408 DEFINE_PROP_UINT8("sriov_max_vfs", NvmeCtrl, params.sriov_max_vfs, 0),
8409 DEFINE_PROP_UINT16("sriov_vq_flexible", NvmeCtrl,
8410 params.sriov_vq_flexible, 0),
8411 DEFINE_PROP_UINT16("sriov_vi_flexible", NvmeCtrl,
8412 params.sriov_vi_flexible, 0),
8413 DEFINE_PROP_UINT8("sriov_max_vi_per_vf", NvmeCtrl,
8414 params.sriov_max_vi_per_vf, 0),
8415 DEFINE_PROP_UINT8("sriov_max_vq_per_vf", NvmeCtrl,
8416 params.sriov_max_vq_per_vf, 0),
8417 DEFINE_PROP_END_OF_LIST(),
8418 };
8419
8420 static void nvme_get_smart_warning(Object *obj, Visitor *v, const char *name,
8421 void *opaque, Error **errp)
8422 {
8423 NvmeCtrl *n = NVME(obj);
8424 uint8_t value = n->smart_critical_warning;
8425
8426 visit_type_uint8(v, name, &value, errp);
8427 }
8428
8429 static void nvme_set_smart_warning(Object *obj, Visitor *v, const char *name,
8430 void *opaque, Error **errp)
8431 {
8432 NvmeCtrl *n = NVME(obj);
8433 uint8_t value, old_value, cap = 0, index, event;
8434
8435 if (!visit_type_uint8(v, name, &value, errp)) {
8436 return;
8437 }
8438
8439 cap = NVME_SMART_SPARE | NVME_SMART_TEMPERATURE | NVME_SMART_RELIABILITY
8440 | NVME_SMART_MEDIA_READ_ONLY | NVME_SMART_FAILED_VOLATILE_MEDIA;
8441 if (NVME_CAP_PMRS(ldq_le_p(&n->bar.cap))) {
8442 cap |= NVME_SMART_PMR_UNRELIABLE;
8443 }
8444
8445 if ((value & cap) != value) {
8446 error_setg(errp, "unsupported smart critical warning bits: 0x%x",
8447 value & ~cap);
8448 return;
8449 }
8450
8451 old_value = n->smart_critical_warning;
8452 n->smart_critical_warning = value;
8453
8454 /* only inject new bits of smart critical warning */
8455 for (index = 0; index < NVME_SMART_WARN_MAX; index++) {
8456 event = 1 << index;
8457 if (value & ~old_value & event)
8458 nvme_smart_event(n, event);
8459 }
8460 }
8461
8462 static void nvme_pci_reset(DeviceState *qdev)
8463 {
8464 PCIDevice *pci_dev = PCI_DEVICE(qdev);
8465 NvmeCtrl *n = NVME(pci_dev);
8466
8467 trace_pci_nvme_pci_reset();
8468 nvme_ctrl_reset(n, NVME_RESET_FUNCTION);
8469 }
8470
8471 static void nvme_sriov_pre_write_ctrl(PCIDevice *dev, uint32_t address,
8472 uint32_t val, int len)
8473 {
8474 NvmeCtrl *n = NVME(dev);
8475 NvmeSecCtrlEntry *sctrl;
8476 uint16_t sriov_cap = dev->exp.sriov_cap;
8477 uint32_t off = address - sriov_cap;
8478 int i, num_vfs;
8479
8480 if (!sriov_cap) {
8481 return;
8482 }
8483
8484 if (range_covers_byte(off, len, PCI_SRIOV_CTRL)) {
8485 if (!(val & PCI_SRIOV_CTRL_VFE)) {
8486 num_vfs = pci_get_word(dev->config + sriov_cap + PCI_SRIOV_NUM_VF);
8487 for (i = 0; i < num_vfs; i++) {
8488 sctrl = &n->sec_ctrl_list.sec[i];
8489 nvme_virt_set_state(n, le16_to_cpu(sctrl->scid), false);
8490 }
8491 }
8492 }
8493 }
8494
8495 static void nvme_pci_write_config(PCIDevice *dev, uint32_t address,
8496 uint32_t val, int len)
8497 {
8498 nvme_sriov_pre_write_ctrl(dev, address, val, len);
8499 pci_default_write_config(dev, address, val, len);
8500 pcie_cap_flr_write_config(dev, address, val, len);
8501 }
8502
8503 static const VMStateDescription nvme_vmstate = {
8504 .name = "nvme",
8505 .unmigratable = 1,
8506 };
8507
8508 static void nvme_class_init(ObjectClass *oc, void *data)
8509 {
8510 DeviceClass *dc = DEVICE_CLASS(oc);
8511 PCIDeviceClass *pc = PCI_DEVICE_CLASS(oc);
8512
8513 pc->realize = nvme_realize;
8514 pc->config_write = nvme_pci_write_config;
8515 pc->exit = nvme_exit;
8516 pc->class_id = PCI_CLASS_STORAGE_EXPRESS;
8517 pc->revision = 2;
8518
8519 set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
8520 dc->desc = "Non-Volatile Memory Express";
8521 device_class_set_props(dc, nvme_props);
8522 dc->vmsd = &nvme_vmstate;
8523 dc->reset = nvme_pci_reset;
8524 }
8525
8526 static void nvme_instance_init(Object *obj)
8527 {
8528 NvmeCtrl *n = NVME(obj);
8529
8530 device_add_bootindex_property(obj, &n->namespace.blkconf.bootindex,
8531 "bootindex", "/namespace@1,0",
8532 DEVICE(obj));
8533
8534 object_property_add(obj, "smart_critical_warning", "uint8",
8535 nvme_get_smart_warning,
8536 nvme_set_smart_warning, NULL, NULL);
8537 }
8538
8539 static const TypeInfo nvme_info = {
8540 .name = TYPE_NVME,
8541 .parent = TYPE_PCI_DEVICE,
8542 .instance_size = sizeof(NvmeCtrl),
8543 .instance_init = nvme_instance_init,
8544 .class_init = nvme_class_init,
8545 .interfaces = (InterfaceInfo[]) {
8546 { INTERFACE_PCIE_DEVICE },
8547 { }
8548 },
8549 };
8550
8551 static const TypeInfo nvme_bus_info = {
8552 .name = TYPE_NVME_BUS,
8553 .parent = TYPE_BUS,
8554 .instance_size = sizeof(NvmeBus),
8555 };
8556
8557 static void nvme_register_types(void)
8558 {
8559 type_register_static(&nvme_info);
8560 type_register_static(&nvme_bus_info);
8561 }
8562
8563 type_init(nvme_register_types)
QEMU mirror