]> git.proxmox.com Git - mirror_qemu.git/blob - hw/nvme/ctrl.c
projects / mirror_qemu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
net/vhost-net: do not assert on null pointer return from tap_get_vhost_net()
[mirror_qemu.git] / hw / nvme / ctrl.c
1 /*
2 * QEMU NVM Express Controller
3 *
4 * Copyright (c) 2012, Intel Corporation
5 *
6 * Written by Keith Busch <keith.busch@intel.com>
7 *
8 * This code is licensed under the GNU GPL v2 or later.
9 */
10
11 /**
12 * Reference Specs: http://www.nvmexpress.org, 1.4, 1.3, 1.2, 1.1, 1.0e
13 *
14 * https://nvmexpress.org/developers/nvme-specification/
15 *
16 *
17 * Notes on coding style
18 * ---------------------
19 * While QEMU coding style prefers lowercase hexadecimals in constants, the
20 * NVMe subsystem use thes format from the NVMe specifications in the comments
21 * (i.e. 'h' suffix instead of '0x' prefix).
22 *
23 * Usage
24 * -----
25 * See docs/system/nvme.rst for extensive documentation.
26 *
27 * Add options:
28 * -drive file=<file>,if=none,id=<drive_id>
29 * -device nvme-subsys,id=<subsys_id>,nqn=<nqn_id>
30 * -device nvme,serial=<serial>,id=<bus_name>, \
31 * cmb_size_mb=<cmb_size_mb[optional]>, \
32 * [pmrdev=<mem_backend_file_id>,] \
33 * max_ioqpairs=<N[optional]>, \
34 * aerl=<N[optional]>,aer_max_queued=<N[optional]>, \
35 * mdts=<N[optional]>,vsl=<N[optional]>, \
36 * zoned.zasl=<N[optional]>, \
37 * zoned.auto_transition=<on|off[optional]>, \
38 * sriov_max_vfs=<N[optional]> \
39 * sriov_vq_flexible=<N[optional]> \
40 * sriov_vi_flexible=<N[optional]> \
41 * sriov_max_vi_per_vf=<N[optional]> \
42 * sriov_max_vq_per_vf=<N[optional]> \
43 * subsys=<subsys_id>
44 * -device nvme-ns,drive=<drive_id>,bus=<bus_name>,nsid=<nsid>,\
45 * zoned=<true|false[optional]>, \
46 * subsys=<subsys_id>,detached=<true|false[optional]>
47 *
48 * Note cmb_size_mb denotes size of CMB in MB. CMB is assumed to be at
49 * offset 0 in BAR2 and supports only WDS, RDS and SQS for now. By default, the
50 * device will use the "v1.4 CMB scheme" - use the `legacy-cmb` parameter to
51 * always enable the CMBLOC and CMBSZ registers (v1.3 behavior).
52 *
53 * Enabling pmr emulation can be achieved by pointing to memory-backend-file.
54 * For example:
55 * -object memory-backend-file,id=<mem_id>,share=on,mem-path=<file_path>, \
56 * size=<size> .... -device nvme,...,pmrdev=<mem_id>
57 *
58 * The PMR will use BAR 4/5 exclusively.
59 *
60 * To place controller(s) and namespace(s) to a subsystem, then provide
61 * nvme-subsys device as above.
62 *
63 * nvme subsystem device parameters
64 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
65 * - `nqn`
66 * This parameter provides the `<nqn_id>` part of the string
67 * `nqn.2019-08.org.qemu:<nqn_id>` which will be reported in the SUBNQN field
68 * of subsystem controllers. Note that `<nqn_id>` should be unique per
69 * subsystem, but this is not enforced by QEMU. If not specified, it will
70 * default to the value of the `id` parameter (`<subsys_id>`).
71 *
72 * nvme device parameters
73 * ~~~~~~~~~~~~~~~~~~~~~~
74 * - `subsys`
75 * Specifying this parameter attaches the controller to the subsystem and
76 * the SUBNQN field in the controller will report the NQN of the subsystem
77 * device. This also enables multi controller capability represented in
78 * Identify Controller data structure in CMIC (Controller Multi-path I/O and
79 * Namespace Sharing Capabilities).
80 *
81 * - `aerl`
82 * The Asynchronous Event Request Limit (AERL). Indicates the maximum number
83 * of concurrently outstanding Asynchronous Event Request commands support
84 * by the controller. This is a 0's based value.
85 *
86 * - `aer_max_queued`
87 * This is the maximum number of events that the device will enqueue for
88 * completion when there are no outstanding AERs. When the maximum number of
89 * enqueued events are reached, subsequent events will be dropped.
90 *
91 * - `mdts`
92 * Indicates the maximum data transfer size for a command that transfers data
93 * between host-accessible memory and the controller. The value is specified
94 * as a power of two (2^n) and is in units of the minimum memory page size
95 * (CAP.MPSMIN). The default value is 7 (i.e. 512 KiB).
96 *
97 * - `vsl`
98 * Indicates the maximum data size limit for the Verify command. Like `mdts`,
99 * this value is specified as a power of two (2^n) and is in units of the
100 * minimum memory page size (CAP.MPSMIN). The default value is 7 (i.e. 512
101 * KiB).
102 *
103 * - `zoned.zasl`
104 * Indicates the maximum data transfer size for the Zone Append command. Like
105 * `mdts`, the value is specified as a power of two (2^n) and is in units of
106 * the minimum memory page size (CAP.MPSMIN). The default value is 0 (i.e.
107 * defaulting to the value of `mdts`).
108 *
109 * - `zoned.auto_transition`
110 * Indicates if zones in zone state implicitly opened can be automatically
111 * transitioned to zone state closed for resource management purposes.
112 * Defaults to 'on'.
113 *
114 * - `sriov_max_vfs`
115 * Indicates the maximum number of PCIe virtual functions supported
116 * by the controller. The default value is 0. Specifying a non-zero value
117 * enables reporting of both SR-IOV and ARI capabilities by the NVMe device.
118 * Virtual function controllers will not report SR-IOV capability.
119 *
120 * NOTE: Single Root I/O Virtualization support is experimental.
121 * All the related parameters may be subject to change.
122 *
123 * - `sriov_vq_flexible`
124 * Indicates the total number of flexible queue resources assignable to all
125 * the secondary controllers. Implicitly sets the number of primary
126 * controller's private resources to `(max_ioqpairs - sriov_vq_flexible)`.
127 *
128 * - `sriov_vi_flexible`
129 * Indicates the total number of flexible interrupt resources assignable to
130 * all the secondary controllers. Implicitly sets the number of primary
131 * controller's private resources to `(msix_qsize - sriov_vi_flexible)`.
132 *
133 * - `sriov_max_vi_per_vf`
134 * Indicates the maximum number of virtual interrupt resources assignable
135 * to a secondary controller. The default 0 resolves to
136 * `(sriov_vi_flexible / sriov_max_vfs)`.
137 *
138 * - `sriov_max_vq_per_vf`
139 * Indicates the maximum number of virtual queue resources assignable to
140 * a secondary controller. The default 0 resolves to
141 * `(sriov_vq_flexible / sriov_max_vfs)`.
142 *
143 * nvme namespace device parameters
144 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
145 * - `shared`
146 * When the parent nvme device (as defined explicitly by the 'bus' parameter
147 * or implicitly by the most recently defined NvmeBus) is linked to an
148 * nvme-subsys device, the namespace will be attached to all controllers in
149 * the subsystem. If set to 'off' (the default), the namespace will remain a
150 * private namespace and may only be attached to a single controller at a
151 * time.
152 *
153 * - `detached`
154 * This parameter is only valid together with the `subsys` parameter. If left
155 * at the default value (`false/off`), the namespace will be attached to all
156 * controllers in the NVMe subsystem at boot-up. If set to `true/on`, the
157 * namespace will be available in the subsystem but not attached to any
158 * controllers.
159 *
160 * Setting `zoned` to true selects Zoned Command Set at the namespace.
161 * In this case, the following namespace properties are available to configure
162 * zoned operation:
163 * zoned.zone_size=<zone size in bytes, default: 128MiB>
164 * The number may be followed by K, M, G as in kilo-, mega- or giga-.
165 *
166 * zoned.zone_capacity=<zone capacity in bytes, default: zone size>
167 * The value 0 (default) forces zone capacity to be the same as zone
168 * size. The value of this property may not exceed zone size.
169 *
170 * zoned.descr_ext_size=<zone descriptor extension size, default 0>
171 * This value needs to be specified in 64B units. If it is zero,
172 * namespace(s) will not support zone descriptor extensions.
173 *
174 * zoned.max_active=<Maximum Active Resources (zones), default: 0>
175 * The default value means there is no limit to the number of
176 * concurrently active zones.
177 *
178 * zoned.max_open=<Maximum Open Resources (zones), default: 0>
179 * The default value means there is no limit to the number of
180 * concurrently open zones.
181 *
182 * zoned.cross_read=<enable RAZB, default: false>
183 * Setting this property to true enables Read Across Zone Boundaries.
184 */
185
186 #include "qemu/osdep.h"
187 #include "qemu/cutils.h"
188 #include "qemu/error-report.h"
189 #include "qemu/log.h"
190 #include "qemu/units.h"
191 #include "qemu/range.h"
192 #include "qapi/error.h"
193 #include "qapi/visitor.h"
194 #include "sysemu/sysemu.h"
195 #include "sysemu/block-backend.h"
196 #include "sysemu/hostmem.h"
197 #include "hw/pci/msix.h"
198 #include "hw/pci/pcie_sriov.h"
199 #include "migration/vmstate.h"
200
201 #include "nvme.h"
202 #include "dif.h"
203 #include "trace.h"
204
205 #define NVME_MAX_IOQPAIRS 0xffff
206 #define NVME_DB_SIZE 4
207 #define NVME_SPEC_VER 0x00010400
208 #define NVME_CMB_BIR 2
209 #define NVME_PMR_BIR 4
210 #define NVME_TEMPERATURE 0x143
211 #define NVME_TEMPERATURE_WARNING 0x157
212 #define NVME_TEMPERATURE_CRITICAL 0x175
213 #define NVME_NUM_FW_SLOTS 1
214 #define NVME_DEFAULT_MAX_ZA_SIZE (128 * KiB)
215 #define NVME_MAX_VFS 127
216 #define NVME_VF_RES_GRANULARITY 1
217 #define NVME_VF_OFFSET 0x1
218 #define NVME_VF_STRIDE 1
219
220 #define NVME_GUEST_ERR(trace, fmt, ...) \
221 do { \
222 (trace_##trace)(__VA_ARGS__); \
223 qemu_log_mask(LOG_GUEST_ERROR, #trace \
224 " in %s: " fmt "\n", __func__, ## __VA_ARGS__); \
225 } while (0)
226
227 static const bool nvme_feature_support[NVME_FID_MAX] = {
228 [NVME_ARBITRATION] = true,
229 [NVME_POWER_MANAGEMENT] = true,
230 [NVME_TEMPERATURE_THRESHOLD] = true,
231 [NVME_ERROR_RECOVERY] = true,
232 [NVME_VOLATILE_WRITE_CACHE] = true,
233 [NVME_NUMBER_OF_QUEUES] = true,
234 [NVME_INTERRUPT_COALESCING] = true,
235 [NVME_INTERRUPT_VECTOR_CONF] = true,
236 [NVME_WRITE_ATOMICITY] = true,
237 [NVME_ASYNCHRONOUS_EVENT_CONF] = true,
238 [NVME_TIMESTAMP] = true,
239 [NVME_HOST_BEHAVIOR_SUPPORT] = true,
240 [NVME_COMMAND_SET_PROFILE] = true,
241 [NVME_FDP_MODE] = true,
242 [NVME_FDP_EVENTS] = true,
243 };
244
245 static const uint32_t nvme_feature_cap[NVME_FID_MAX] = {
246 [NVME_TEMPERATURE_THRESHOLD] = NVME_FEAT_CAP_CHANGE,
247 [NVME_ERROR_RECOVERY] = NVME_FEAT_CAP_CHANGE | NVME_FEAT_CAP_NS,
248 [NVME_VOLATILE_WRITE_CACHE] = NVME_FEAT_CAP_CHANGE,
249 [NVME_NUMBER_OF_QUEUES] = NVME_FEAT_CAP_CHANGE,
250 [NVME_ASYNCHRONOUS_EVENT_CONF] = NVME_FEAT_CAP_CHANGE,
251 [NVME_TIMESTAMP] = NVME_FEAT_CAP_CHANGE,
252 [NVME_HOST_BEHAVIOR_SUPPORT] = NVME_FEAT_CAP_CHANGE,
253 [NVME_COMMAND_SET_PROFILE] = NVME_FEAT_CAP_CHANGE,
254 [NVME_FDP_MODE] = NVME_FEAT_CAP_CHANGE,
255 [NVME_FDP_EVENTS] = NVME_FEAT_CAP_CHANGE | NVME_FEAT_CAP_NS,
256 };
257
258 static const uint32_t nvme_cse_acs[256] = {
259 [NVME_ADM_CMD_DELETE_SQ] = NVME_CMD_EFF_CSUPP,
260 [NVME_ADM_CMD_CREATE_SQ] = NVME_CMD_EFF_CSUPP,
261 [NVME_ADM_CMD_GET_LOG_PAGE] = NVME_CMD_EFF_CSUPP,
262 [NVME_ADM_CMD_DELETE_CQ] = NVME_CMD_EFF_CSUPP,
263 [NVME_ADM_CMD_CREATE_CQ] = NVME_CMD_EFF_CSUPP,
264 [NVME_ADM_CMD_IDENTIFY] = NVME_CMD_EFF_CSUPP,
265 [NVME_ADM_CMD_ABORT] = NVME_CMD_EFF_CSUPP,
266 [NVME_ADM_CMD_SET_FEATURES] = NVME_CMD_EFF_CSUPP,
267 [NVME_ADM_CMD_GET_FEATURES] = NVME_CMD_EFF_CSUPP,
268 [NVME_ADM_CMD_ASYNC_EV_REQ] = NVME_CMD_EFF_CSUPP,
269 [NVME_ADM_CMD_NS_ATTACHMENT] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_NIC,
270 [NVME_ADM_CMD_VIRT_MNGMT] = NVME_CMD_EFF_CSUPP,
271 [NVME_ADM_CMD_DBBUF_CONFIG] = NVME_CMD_EFF_CSUPP,
272 [NVME_ADM_CMD_FORMAT_NVM] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
273 [NVME_ADM_CMD_DIRECTIVE_RECV] = NVME_CMD_EFF_CSUPP,
274 [NVME_ADM_CMD_DIRECTIVE_SEND] = NVME_CMD_EFF_CSUPP,
275 };
276
277 static const uint32_t nvme_cse_iocs_none[256];
278
279 static const uint32_t nvme_cse_iocs_nvm[256] = {
280 [NVME_CMD_FLUSH] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
281 [NVME_CMD_WRITE_ZEROES] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
282 [NVME_CMD_WRITE] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
283 [NVME_CMD_READ] = NVME_CMD_EFF_CSUPP,
284 [NVME_CMD_DSM] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
285 [NVME_CMD_VERIFY] = NVME_CMD_EFF_CSUPP,
286 [NVME_CMD_COPY] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
287 [NVME_CMD_COMPARE] = NVME_CMD_EFF_CSUPP,
288 [NVME_CMD_IO_MGMT_RECV] = NVME_CMD_EFF_CSUPP,
289 [NVME_CMD_IO_MGMT_SEND] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
290 };
291
292 static const uint32_t nvme_cse_iocs_zoned[256] = {
293 [NVME_CMD_FLUSH] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
294 [NVME_CMD_WRITE_ZEROES] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
295 [NVME_CMD_WRITE] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
296 [NVME_CMD_READ] = NVME_CMD_EFF_CSUPP,
297 [NVME_CMD_DSM] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
298 [NVME_CMD_VERIFY] = NVME_CMD_EFF_CSUPP,
299 [NVME_CMD_COPY] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
300 [NVME_CMD_COMPARE] = NVME_CMD_EFF_CSUPP,
301 [NVME_CMD_ZONE_APPEND] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
302 [NVME_CMD_ZONE_MGMT_SEND] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
303 [NVME_CMD_ZONE_MGMT_RECV] = NVME_CMD_EFF_CSUPP,
304 };
305
306 static void nvme_process_sq(void *opaque);
307 static void nvme_ctrl_reset(NvmeCtrl *n, NvmeResetType rst);
308 static inline uint64_t nvme_get_timestamp(const NvmeCtrl *n);
309
310 static uint16_t nvme_sqid(NvmeRequest *req)
311 {
312 return le16_to_cpu(req->sq->sqid);
313 }
314
315 static inline uint16_t nvme_make_pid(NvmeNamespace *ns, uint16_t rg,
316 uint16_t ph)
317 {
318 uint16_t rgif = ns->endgrp->fdp.rgif;
319
320 if (!rgif) {
321 return ph;
322 }
323
324 return (rg << (16 - rgif)) | ph;
325 }
326
327 static inline bool nvme_ph_valid(NvmeNamespace *ns, uint16_t ph)
328 {
329 return ph < ns->fdp.nphs;
330 }
331
332 static inline bool nvme_rg_valid(NvmeEnduranceGroup *endgrp, uint16_t rg)
333 {
334 return rg < endgrp->fdp.nrg;
335 }
336
337 static inline uint16_t nvme_pid2ph(NvmeNamespace *ns, uint16_t pid)
338 {
339 uint16_t rgif = ns->endgrp->fdp.rgif;
340
341 if (!rgif) {
342 return pid;
343 }
344
345 return pid & ((1 << (15 - rgif)) - 1);
346 }
347
348 static inline uint16_t nvme_pid2rg(NvmeNamespace *ns, uint16_t pid)
349 {
350 uint16_t rgif = ns->endgrp->fdp.rgif;
351
352 if (!rgif) {
353 return 0;
354 }
355
356 return pid >> (16 - rgif);
357 }
358
359 static inline bool nvme_parse_pid(NvmeNamespace *ns, uint16_t pid,
360 uint16_t *ph, uint16_t *rg)
361 {
362 *rg = nvme_pid2rg(ns, pid);
363 *ph = nvme_pid2ph(ns, pid);
364
365 return nvme_ph_valid(ns, *ph) && nvme_rg_valid(ns->endgrp, *rg);
366 }
367
368 static void nvme_assign_zone_state(NvmeNamespace *ns, NvmeZone *zone,
369 NvmeZoneState state)
370 {
371 if (QTAILQ_IN_USE(zone, entry)) {
372 switch (nvme_get_zone_state(zone)) {
373 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
374 QTAILQ_REMOVE(&ns->exp_open_zones, zone, entry);
375 break;
376 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
377 QTAILQ_REMOVE(&ns->imp_open_zones, zone, entry);
378 break;
379 case NVME_ZONE_STATE_CLOSED:
380 QTAILQ_REMOVE(&ns->closed_zones, zone, entry);
381 break;
382 case NVME_ZONE_STATE_FULL:
383 QTAILQ_REMOVE(&ns->full_zones, zone, entry);
384 default:
385 ;
386 }
387 }
388
389 nvme_set_zone_state(zone, state);
390
391 switch (state) {
392 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
393 QTAILQ_INSERT_TAIL(&ns->exp_open_zones, zone, entry);
394 break;
395 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
396 QTAILQ_INSERT_TAIL(&ns->imp_open_zones, zone, entry);
397 break;
398 case NVME_ZONE_STATE_CLOSED:
399 QTAILQ_INSERT_TAIL(&ns->closed_zones, zone, entry);
400 break;
401 case NVME_ZONE_STATE_FULL:
402 QTAILQ_INSERT_TAIL(&ns->full_zones, zone, entry);
403 case NVME_ZONE_STATE_READ_ONLY:
404 break;
405 default:
406 zone->d.za = 0;
407 }
408 }
409
410 static uint16_t nvme_zns_check_resources(NvmeNamespace *ns, uint32_t act,
411 uint32_t opn, uint32_t zrwa)
412 {
413 if (ns->params.max_active_zones != 0 &&
414 ns->nr_active_zones + act > ns->params.max_active_zones) {
415 trace_pci_nvme_err_insuff_active_res(ns->params.max_active_zones);
416 return NVME_ZONE_TOO_MANY_ACTIVE | NVME_DNR;
417 }
418
419 if (ns->params.max_open_zones != 0 &&
420 ns->nr_open_zones + opn > ns->params.max_open_zones) {
421 trace_pci_nvme_err_insuff_open_res(ns->params.max_open_zones);
422 return NVME_ZONE_TOO_MANY_OPEN | NVME_DNR;
423 }
424
425 if (zrwa > ns->zns.numzrwa) {
426 return NVME_NOZRWA | NVME_DNR;
427 }
428
429 return NVME_SUCCESS;
430 }
431
432 /*
433 * Check if we can open a zone without exceeding open/active limits.
434 * AOR stands for "Active and Open Resources" (see TP 4053 section 2.5).
435 */
436 static uint16_t nvme_aor_check(NvmeNamespace *ns, uint32_t act, uint32_t opn)
437 {
438 return nvme_zns_check_resources(ns, act, opn, 0);
439 }
440
441 static NvmeFdpEvent *nvme_fdp_alloc_event(NvmeCtrl *n, NvmeFdpEventBuffer *ebuf)
442 {
443 NvmeFdpEvent *ret = NULL;
444 bool is_full = ebuf->next == ebuf->start && ebuf->nelems;
445
446 ret = &ebuf->events[ebuf->next++];
447 if (unlikely(ebuf->next == NVME_FDP_MAX_EVENTS)) {
448 ebuf->next = 0;
449 }
450 if (is_full) {
451 ebuf->start = ebuf->next;
452 } else {
453 ebuf->nelems++;
454 }
455
456 memset(ret, 0, sizeof(NvmeFdpEvent));
457 ret->timestamp = nvme_get_timestamp(n);
458
459 return ret;
460 }
461
462 static inline int log_event(NvmeRuHandle *ruh, uint8_t event_type)
463 {
464 return (ruh->event_filter >> nvme_fdp_evf_shifts[event_type]) & 0x1;
465 }
466
467 static bool nvme_update_ruh(NvmeCtrl *n, NvmeNamespace *ns, uint16_t pid)
468 {
469 NvmeEnduranceGroup *endgrp = ns->endgrp;
470 NvmeRuHandle *ruh;
471 NvmeReclaimUnit *ru;
472 NvmeFdpEvent *e = NULL;
473 uint16_t ph, rg, ruhid;
474
475 if (!nvme_parse_pid(ns, pid, &ph, &rg)) {
476 return false;
477 }
478
479 ruhid = ns->fdp.phs[ph];
480
481 ruh = &endgrp->fdp.ruhs[ruhid];
482 ru = &ruh->rus[rg];
483
484 if (ru->ruamw) {
485 if (log_event(ruh, FDP_EVT_RU_NOT_FULLY_WRITTEN)) {
486 e = nvme_fdp_alloc_event(n, &endgrp->fdp.host_events);
487 e->type = FDP_EVT_RU_NOT_FULLY_WRITTEN;
488 e->flags = FDPEF_PIV | FDPEF_NSIDV | FDPEF_LV;
489 e->pid = cpu_to_le16(pid);
490 e->nsid = cpu_to_le32(ns->params.nsid);
491 e->rgid = cpu_to_le16(rg);
492 e->ruhid = cpu_to_le16(ruhid);
493 }
494
495 /* log (eventual) GC overhead of prematurely swapping the RU */
496 nvme_fdp_stat_inc(&endgrp->fdp.mbmw, nvme_l2b(ns, ru->ruamw));
497 }
498
499 ru->ruamw = ruh->ruamw;
500
501 return true;
502 }
503
504 static bool nvme_addr_is_cmb(NvmeCtrl *n, hwaddr addr)
505 {
506 hwaddr hi, lo;
507
508 if (!n->cmb.cmse) {
509 return false;
510 }
511
512 lo = n->params.legacy_cmb ? n->cmb.mem.addr : n->cmb.cba;
513 hi = lo + int128_get64(n->cmb.mem.size);
514
515 return addr >= lo && addr < hi;
516 }
517
518 static inline void *nvme_addr_to_cmb(NvmeCtrl *n, hwaddr addr)
519 {
520 hwaddr base = n->params.legacy_cmb ? n->cmb.mem.addr : n->cmb.cba;
521 return &n->cmb.buf[addr - base];
522 }
523
524 static bool nvme_addr_is_pmr(NvmeCtrl *n, hwaddr addr)
525 {
526 hwaddr hi;
527
528 if (!n->pmr.cmse) {
529 return false;
530 }
531
532 hi = n->pmr.cba + int128_get64(n->pmr.dev->mr.size);
533
534 return addr >= n->pmr.cba && addr < hi;
535 }
536
537 static inline void *nvme_addr_to_pmr(NvmeCtrl *n, hwaddr addr)
538 {
539 return memory_region_get_ram_ptr(&n->pmr.dev->mr) + (addr - n->pmr.cba);
540 }
541
542 static inline bool nvme_addr_is_iomem(NvmeCtrl *n, hwaddr addr)
543 {
544 hwaddr hi, lo;
545
546 /*
547 * The purpose of this check is to guard against invalid "local" access to
548 * the iomem (i.e. controller registers). Thus, we check against the range
549 * covered by the 'bar0' MemoryRegion since that is currently composed of
550 * two subregions (the NVMe "MBAR" and the MSI-X table/pba). Note, however,
551 * that if the device model is ever changed to allow the CMB to be located
552 * in BAR0 as well, then this must be changed.
553 */
554 lo = n->bar0.addr;
555 hi = lo + int128_get64(n->bar0.size);
556
557 return addr >= lo && addr < hi;
558 }
559
560 static int nvme_addr_read(NvmeCtrl *n, hwaddr addr, void *buf, int size)
561 {
562 hwaddr hi = addr + size - 1;
563 if (hi < addr) {
564 return 1;
565 }
566
567 if (n->bar.cmbsz && nvme_addr_is_cmb(n, addr) && nvme_addr_is_cmb(n, hi)) {
568 memcpy(buf, nvme_addr_to_cmb(n, addr), size);
569 return 0;
570 }
571
572 if (nvme_addr_is_pmr(n, addr) && nvme_addr_is_pmr(n, hi)) {
573 memcpy(buf, nvme_addr_to_pmr(n, addr), size);
574 return 0;
575 }
576
577 return pci_dma_read(PCI_DEVICE(n), addr, buf, size);
578 }
579
580 static int nvme_addr_write(NvmeCtrl *n, hwaddr addr, const void *buf, int size)
581 {
582 hwaddr hi = addr + size - 1;
583 if (hi < addr) {
584 return 1;
585 }
586
587 if (n->bar.cmbsz && nvme_addr_is_cmb(n, addr) && nvme_addr_is_cmb(n, hi)) {
588 memcpy(nvme_addr_to_cmb(n, addr), buf, size);
589 return 0;
590 }
591
592 if (nvme_addr_is_pmr(n, addr) && nvme_addr_is_pmr(n, hi)) {
593 memcpy(nvme_addr_to_pmr(n, addr), buf, size);
594 return 0;
595 }
596
597 return pci_dma_write(PCI_DEVICE(n), addr, buf, size);
598 }
599
600 static bool nvme_nsid_valid(NvmeCtrl *n, uint32_t nsid)
601 {
602 return nsid &&
603 (nsid == NVME_NSID_BROADCAST || nsid <= NVME_MAX_NAMESPACES);
604 }
605
606 static int nvme_check_sqid(NvmeCtrl *n, uint16_t sqid)
607 {
608 return sqid < n->conf_ioqpairs + 1 && n->sq[sqid] != NULL ? 0 : -1;
609 }
610
611 static int nvme_check_cqid(NvmeCtrl *n, uint16_t cqid)
612 {
613 return cqid < n->conf_ioqpairs + 1 && n->cq[cqid] != NULL ? 0 : -1;
614 }
615
616 static void nvme_inc_cq_tail(NvmeCQueue *cq)
617 {
618 cq->tail++;
619 if (cq->tail >= cq->size) {
620 cq->tail = 0;
621 cq->phase = !cq->phase;
622 }
623 }
624
625 static void nvme_inc_sq_head(NvmeSQueue *sq)
626 {
627 sq->head = (sq->head + 1) % sq->size;
628 }
629
630 static uint8_t nvme_cq_full(NvmeCQueue *cq)
631 {
632 return (cq->tail + 1) % cq->size == cq->head;
633 }
634
635 static uint8_t nvme_sq_empty(NvmeSQueue *sq)
636 {
637 return sq->head == sq->tail;
638 }
639
640 static void nvme_irq_check(NvmeCtrl *n)
641 {
642 PCIDevice *pci = PCI_DEVICE(n);
643 uint32_t intms = ldl_le_p(&n->bar.intms);
644
645 if (msix_enabled(pci)) {
646 return;
647 }
648 if (~intms & n->irq_status) {
649 pci_irq_assert(pci);
650 } else {
651 pci_irq_deassert(pci);
652 }
653 }
654
655 static void nvme_irq_assert(NvmeCtrl *n, NvmeCQueue *cq)
656 {
657 PCIDevice *pci = PCI_DEVICE(n);
658
659 if (cq->irq_enabled) {
660 if (msix_enabled(pci)) {
661 trace_pci_nvme_irq_msix(cq->vector);
662 msix_notify(pci, cq->vector);
663 } else {
664 trace_pci_nvme_irq_pin();
665 assert(cq->vector < 32);
666 n->irq_status |= 1 << cq->vector;
667 nvme_irq_check(n);
668 }
669 } else {
670 trace_pci_nvme_irq_masked();
671 }
672 }
673
674 static void nvme_irq_deassert(NvmeCtrl *n, NvmeCQueue *cq)
675 {
676 if (cq->irq_enabled) {
677 if (msix_enabled(PCI_DEVICE(n))) {
678 return;
679 } else {
680 assert(cq->vector < 32);
681 if (!n->cq_pending) {
682 n->irq_status &= ~(1 << cq->vector);
683 }
684 nvme_irq_check(n);
685 }
686 }
687 }
688
689 static void nvme_req_clear(NvmeRequest *req)
690 {
691 req->ns = NULL;
692 req->opaque = NULL;
693 req->aiocb = NULL;
694 memset(&req->cqe, 0x0, sizeof(req->cqe));
695 req->status = NVME_SUCCESS;
696 }
697
698 static inline void nvme_sg_init(NvmeCtrl *n, NvmeSg *sg, bool dma)
699 {
700 if (dma) {
701 pci_dma_sglist_init(&sg->qsg, PCI_DEVICE(n), 0);
702 sg->flags = NVME_SG_DMA;
703 } else {
704 qemu_iovec_init(&sg->iov, 0);
705 }
706
707 sg->flags |= NVME_SG_ALLOC;
708 }
709
710 static inline void nvme_sg_unmap(NvmeSg *sg)
711 {
712 if (!(sg->flags & NVME_SG_ALLOC)) {
713 return;
714 }
715
716 if (sg->flags & NVME_SG_DMA) {
717 qemu_sglist_destroy(&sg->qsg);
718 } else {
719 qemu_iovec_destroy(&sg->iov);
720 }
721
722 memset(sg, 0x0, sizeof(*sg));
723 }
724
725 /*
726 * When metadata is transfered as extended LBAs, the DPTR mapped into `sg`
727 * holds both data and metadata. This function splits the data and metadata
728 * into two separate QSG/IOVs.
729 */
730 static void nvme_sg_split(NvmeSg *sg, NvmeNamespace *ns, NvmeSg *data,
731 NvmeSg *mdata)
732 {
733 NvmeSg *dst = data;
734 uint32_t trans_len, count = ns->lbasz;
735 uint64_t offset = 0;
736 bool dma = sg->flags & NVME_SG_DMA;
737 size_t sge_len;
738 size_t sg_len = dma ? sg->qsg.size : sg->iov.size;
739 int sg_idx = 0;
740
741 assert(sg->flags & NVME_SG_ALLOC);
742
743 while (sg_len) {
744 sge_len = dma ? sg->qsg.sg[sg_idx].len : sg->iov.iov[sg_idx].iov_len;
745
746 trans_len = MIN(sg_len, count);
747 trans_len = MIN(trans_len, sge_len - offset);
748
749 if (dst) {
750 if (dma) {
751 qemu_sglist_add(&dst->qsg, sg->qsg.sg[sg_idx].base + offset,
752 trans_len);
753 } else {
754 qemu_iovec_add(&dst->iov,
755 sg->iov.iov[sg_idx].iov_base + offset,
756 trans_len);
757 }
758 }
759
760 sg_len -= trans_len;
761 count -= trans_len;
762 offset += trans_len;
763
764 if (count == 0) {
765 dst = (dst == data) ? mdata : data;
766 count = (dst == data) ? ns->lbasz : ns->lbaf.ms;
767 }
768
769 if (sge_len == offset) {
770 offset = 0;
771 sg_idx++;
772 }
773 }
774 }
775
776 static uint16_t nvme_map_addr_cmb(NvmeCtrl *n, QEMUIOVector *iov, hwaddr addr,
777 size_t len)
778 {
779 if (!len) {
780 return NVME_SUCCESS;
781 }
782
783 trace_pci_nvme_map_addr_cmb(addr, len);
784
785 if (!nvme_addr_is_cmb(n, addr) || !nvme_addr_is_cmb(n, addr + len - 1)) {
786 return NVME_DATA_TRAS_ERROR;
787 }
788
789 qemu_iovec_add(iov, nvme_addr_to_cmb(n, addr), len);
790
791 return NVME_SUCCESS;
792 }
793
794 static uint16_t nvme_map_addr_pmr(NvmeCtrl *n, QEMUIOVector *iov, hwaddr addr,
795 size_t len)
796 {
797 if (!len) {
798 return NVME_SUCCESS;
799 }
800
801 if (!nvme_addr_is_pmr(n, addr) || !nvme_addr_is_pmr(n, addr + len - 1)) {
802 return NVME_DATA_TRAS_ERROR;
803 }
804
805 qemu_iovec_add(iov, nvme_addr_to_pmr(n, addr), len);
806
807 return NVME_SUCCESS;
808 }
809
810 static uint16_t nvme_map_addr(NvmeCtrl *n, NvmeSg *sg, hwaddr addr, size_t len)
811 {
812 bool cmb = false, pmr = false;
813
814 if (!len) {
815 return NVME_SUCCESS;
816 }
817
818 trace_pci_nvme_map_addr(addr, len);
819
820 if (nvme_addr_is_iomem(n, addr)) {
821 return NVME_DATA_TRAS_ERROR;
822 }
823
824 if (nvme_addr_is_cmb(n, addr)) {
825 cmb = true;
826 } else if (nvme_addr_is_pmr(n, addr)) {
827 pmr = true;
828 }
829
830 if (cmb || pmr) {
831 if (sg->flags & NVME_SG_DMA) {
832 return NVME_INVALID_USE_OF_CMB | NVME_DNR;
833 }
834
835 if (sg->iov.niov + 1 > IOV_MAX) {
836 goto max_mappings_exceeded;
837 }
838
839 if (cmb) {
840 return nvme_map_addr_cmb(n, &sg->iov, addr, len);
841 } else {
842 return nvme_map_addr_pmr(n, &sg->iov, addr, len);
843 }
844 }
845
846 if (!(sg->flags & NVME_SG_DMA)) {
847 return NVME_INVALID_USE_OF_CMB | NVME_DNR;
848 }
849
850 if (sg->qsg.nsg + 1 > IOV_MAX) {
851 goto max_mappings_exceeded;
852 }
853
854 qemu_sglist_add(&sg->qsg, addr, len);
855
856 return NVME_SUCCESS;
857
858 max_mappings_exceeded:
859 NVME_GUEST_ERR(pci_nvme_ub_too_many_mappings,
860 "number of mappings exceed 1024");
861 return NVME_INTERNAL_DEV_ERROR | NVME_DNR;
862 }
863
864 static inline bool nvme_addr_is_dma(NvmeCtrl *n, hwaddr addr)
865 {
866 return !(nvme_addr_is_cmb(n, addr) || nvme_addr_is_pmr(n, addr));
867 }
868
869 static uint16_t nvme_map_prp(NvmeCtrl *n, NvmeSg *sg, uint64_t prp1,
870 uint64_t prp2, uint32_t len)
871 {
872 hwaddr trans_len = n->page_size - (prp1 % n->page_size);
873 trans_len = MIN(len, trans_len);
874 int num_prps = (len >> n->page_bits) + 1;
875 uint16_t status;
876 int ret;
877
878 trace_pci_nvme_map_prp(trans_len, len, prp1, prp2, num_prps);
879
880 nvme_sg_init(n, sg, nvme_addr_is_dma(n, prp1));
881
882 status = nvme_map_addr(n, sg, prp1, trans_len);
883 if (status) {
884 goto unmap;
885 }
886
887 len -= trans_len;
888 if (len) {
889 if (len > n->page_size) {
890 uint64_t prp_list[n->max_prp_ents];
891 uint32_t nents, prp_trans;
892 int i = 0;
893
894 /*
895 * The first PRP list entry, pointed to by PRP2 may contain offset.
896 * Hence, we need to calculate the number of entries in based on
897 * that offset.
898 */
899 nents = (n->page_size - (prp2 & (n->page_size - 1))) >> 3;
900 prp_trans = MIN(n->max_prp_ents, nents) * sizeof(uint64_t);
901 ret = nvme_addr_read(n, prp2, (void *)prp_list, prp_trans);
902 if (ret) {
903 trace_pci_nvme_err_addr_read(prp2);
904 status = NVME_DATA_TRAS_ERROR;
905 goto unmap;
906 }
907 while (len != 0) {
908 uint64_t prp_ent = le64_to_cpu(prp_list[i]);
909
910 if (i == nents - 1 && len > n->page_size) {
911 if (unlikely(prp_ent & (n->page_size - 1))) {
912 trace_pci_nvme_err_invalid_prplist_ent(prp_ent);
913 status = NVME_INVALID_PRP_OFFSET | NVME_DNR;
914 goto unmap;
915 }
916
917 i = 0;
918 nents = (len + n->page_size - 1) >> n->page_bits;
919 nents = MIN(nents, n->max_prp_ents);
920 prp_trans = nents * sizeof(uint64_t);
921 ret = nvme_addr_read(n, prp_ent, (void *)prp_list,
922 prp_trans);
923 if (ret) {
924 trace_pci_nvme_err_addr_read(prp_ent);
925 status = NVME_DATA_TRAS_ERROR;
926 goto unmap;
927 }
928 prp_ent = le64_to_cpu(prp_list[i]);
929 }
930
931 if (unlikely(prp_ent & (n->page_size - 1))) {
932 trace_pci_nvme_err_invalid_prplist_ent(prp_ent);
933 status = NVME_INVALID_PRP_OFFSET | NVME_DNR;
934 goto unmap;
935 }
936
937 trans_len = MIN(len, n->page_size);
938 status = nvme_map_addr(n, sg, prp_ent, trans_len);
939 if (status) {
940 goto unmap;
941 }
942
943 len -= trans_len;
944 i++;
945 }
946 } else {
947 if (unlikely(prp2 & (n->page_size - 1))) {
948 trace_pci_nvme_err_invalid_prp2_align(prp2);
949 status = NVME_INVALID_PRP_OFFSET | NVME_DNR;
950 goto unmap;
951 }
952 status = nvme_map_addr(n, sg, prp2, len);
953 if (status) {
954 goto unmap;
955 }
956 }
957 }
958
959 return NVME_SUCCESS;
960
961 unmap:
962 nvme_sg_unmap(sg);
963 return status;
964 }
965
966 /*
967 * Map 'nsgld' data descriptors from 'segment'. The function will subtract the
968 * number of bytes mapped in len.
969 */
970 static uint16_t nvme_map_sgl_data(NvmeCtrl *n, NvmeSg *sg,
971 NvmeSglDescriptor *segment, uint64_t nsgld,
972 size_t *len, NvmeCmd *cmd)
973 {
974 dma_addr_t addr, trans_len;
975 uint32_t dlen;
976 uint16_t status;
977
978 for (int i = 0; i < nsgld; i++) {
979 uint8_t type = NVME_SGL_TYPE(segment[i].type);
980
981 switch (type) {
982 case NVME_SGL_DESCR_TYPE_DATA_BLOCK:
983 break;
984 case NVME_SGL_DESCR_TYPE_SEGMENT:
985 case NVME_SGL_DESCR_TYPE_LAST_SEGMENT:
986 return NVME_INVALID_NUM_SGL_DESCRS | NVME_DNR;
987 default:
988 return NVME_SGL_DESCR_TYPE_INVALID | NVME_DNR;
989 }
990
991 dlen = le32_to_cpu(segment[i].len);
992
993 if (!dlen) {
994 continue;
995 }
996
997 if (*len == 0) {
998 /*
999 * All data has been mapped, but the SGL contains additional
1000 * segments and/or descriptors. The controller might accept
1001 * ignoring the rest of the SGL.
1002 */
1003 uint32_t sgls = le32_to_cpu(n->id_ctrl.sgls);
1004 if (sgls & NVME_CTRL_SGLS_EXCESS_LENGTH) {
1005 break;
1006 }
1007
1008 trace_pci_nvme_err_invalid_sgl_excess_length(dlen);
1009 return NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
1010 }
1011
1012 trans_len = MIN(*len, dlen);
1013
1014 addr = le64_to_cpu(segment[i].addr);
1015
1016 if (UINT64_MAX - addr < dlen) {
1017 return NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
1018 }
1019
1020 status = nvme_map_addr(n, sg, addr, trans_len);
1021 if (status) {
1022 return status;
1023 }
1024
1025 *len -= trans_len;
1026 }
1027
1028 return NVME_SUCCESS;
1029 }
1030
1031 static uint16_t nvme_map_sgl(NvmeCtrl *n, NvmeSg *sg, NvmeSglDescriptor sgl,
1032 size_t len, NvmeCmd *cmd)
1033 {
1034 /*
1035 * Read the segment in chunks of 256 descriptors (one 4k page) to avoid
1036 * dynamically allocating a potentially huge SGL. The spec allows the SGL
1037 * to be larger (as in number of bytes required to describe the SGL
1038 * descriptors and segment chain) than the command transfer size, so it is
1039 * not bounded by MDTS.
1040 */
1041 const int SEG_CHUNK_SIZE = 256;
1042
1043 NvmeSglDescriptor segment[SEG_CHUNK_SIZE], *sgld, *last_sgld;
1044 uint64_t nsgld;
1045 uint32_t seg_len;
1046 uint16_t status;
1047 hwaddr addr;
1048 int ret;
1049
1050 sgld = &sgl;
1051 addr = le64_to_cpu(sgl.addr);
1052
1053 trace_pci_nvme_map_sgl(NVME_SGL_TYPE(sgl.type), len);
1054
1055 nvme_sg_init(n, sg, nvme_addr_is_dma(n, addr));
1056
1057 /*
1058 * If the entire transfer can be described with a single data block it can
1059 * be mapped directly.
1060 */
1061 if (NVME_SGL_TYPE(sgl.type) == NVME_SGL_DESCR_TYPE_DATA_BLOCK) {
1062 status = nvme_map_sgl_data(n, sg, sgld, 1, &len, cmd);
1063 if (status) {
1064 goto unmap;
1065 }
1066
1067 goto out;
1068 }
1069
1070 for (;;) {
1071 switch (NVME_SGL_TYPE(sgld->type)) {
1072 case NVME_SGL_DESCR_TYPE_SEGMENT:
1073 case NVME_SGL_DESCR_TYPE_LAST_SEGMENT:
1074 break;
1075 default:
1076 return NVME_INVALID_SGL_SEG_DESCR | NVME_DNR;
1077 }
1078
1079 seg_len = le32_to_cpu(sgld->len);
1080
1081 /* check the length of the (Last) Segment descriptor */
1082 if (!seg_len || seg_len & 0xf) {
1083 return NVME_INVALID_SGL_SEG_DESCR | NVME_DNR;
1084 }
1085
1086 if (UINT64_MAX - addr < seg_len) {
1087 return NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
1088 }
1089
1090 nsgld = seg_len / sizeof(NvmeSglDescriptor);
1091
1092 while (nsgld > SEG_CHUNK_SIZE) {
1093 if (nvme_addr_read(n, addr, segment, sizeof(segment))) {
1094 trace_pci_nvme_err_addr_read(addr);
1095 status = NVME_DATA_TRAS_ERROR;
1096 goto unmap;
1097 }
1098
1099 status = nvme_map_sgl_data(n, sg, segment, SEG_CHUNK_SIZE,
1100 &len, cmd);
1101 if (status) {
1102 goto unmap;
1103 }
1104
1105 nsgld -= SEG_CHUNK_SIZE;
1106 addr += SEG_CHUNK_SIZE * sizeof(NvmeSglDescriptor);
1107 }
1108
1109 ret = nvme_addr_read(n, addr, segment, nsgld *
1110 sizeof(NvmeSglDescriptor));
1111 if (ret) {
1112 trace_pci_nvme_err_addr_read(addr);
1113 status = NVME_DATA_TRAS_ERROR;
1114 goto unmap;
1115 }
1116
1117 last_sgld = &segment[nsgld - 1];
1118
1119 /*
1120 * If the segment ends with a Data Block, then we are done.
1121 */
1122 if (NVME_SGL_TYPE(last_sgld->type) == NVME_SGL_DESCR_TYPE_DATA_BLOCK) {
1123 status = nvme_map_sgl_data(n, sg, segment, nsgld, &len, cmd);
1124 if (status) {
1125 goto unmap;
1126 }
1127
1128 goto out;
1129 }
1130
1131 /*
1132 * If the last descriptor was not a Data Block, then the current
1133 * segment must not be a Last Segment.
1134 */
1135 if (NVME_SGL_TYPE(sgld->type) == NVME_SGL_DESCR_TYPE_LAST_SEGMENT) {
1136 status = NVME_INVALID_SGL_SEG_DESCR | NVME_DNR;
1137 goto unmap;
1138 }
1139
1140 sgld = last_sgld;
1141 addr = le64_to_cpu(sgld->addr);
1142
1143 /*
1144 * Do not map the last descriptor; it will be a Segment or Last Segment
1145 * descriptor and is handled by the next iteration.
1146 */
1147 status = nvme_map_sgl_data(n, sg, segment, nsgld - 1, &len, cmd);
1148 if (status) {
1149 goto unmap;
1150 }
1151 }
1152
1153 out:
1154 /* if there is any residual left in len, the SGL was too short */
1155 if (len) {
1156 status = NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
1157 goto unmap;
1158 }
1159
1160 return NVME_SUCCESS;
1161
1162 unmap:
1163 nvme_sg_unmap(sg);
1164 return status;
1165 }
1166
1167 uint16_t nvme_map_dptr(NvmeCtrl *n, NvmeSg *sg, size_t len,
1168 NvmeCmd *cmd)
1169 {
1170 uint64_t prp1, prp2;
1171
1172 switch (NVME_CMD_FLAGS_PSDT(cmd->flags)) {
1173 case NVME_PSDT_PRP:
1174 prp1 = le64_to_cpu(cmd->dptr.prp1);
1175 prp2 = le64_to_cpu(cmd->dptr.prp2);
1176
1177 return nvme_map_prp(n, sg, prp1, prp2, len);
1178 case NVME_PSDT_SGL_MPTR_CONTIGUOUS:
1179 case NVME_PSDT_SGL_MPTR_SGL:
1180 return nvme_map_sgl(n, sg, cmd->dptr.sgl, len, cmd);
1181 default:
1182 return NVME_INVALID_FIELD;
1183 }
1184 }
1185
1186 static uint16_t nvme_map_mptr(NvmeCtrl *n, NvmeSg *sg, size_t len,
1187 NvmeCmd *cmd)
1188 {
1189 int psdt = NVME_CMD_FLAGS_PSDT(cmd->flags);
1190 hwaddr mptr = le64_to_cpu(cmd->mptr);
1191 uint16_t status;
1192
1193 if (psdt == NVME_PSDT_SGL_MPTR_SGL) {
1194 NvmeSglDescriptor sgl;
1195
1196 if (nvme_addr_read(n, mptr, &sgl, sizeof(sgl))) {
1197 return NVME_DATA_TRAS_ERROR;
1198 }
1199
1200 status = nvme_map_sgl(n, sg, sgl, len, cmd);
1201 if (status && (status & 0x7ff) == NVME_DATA_SGL_LEN_INVALID) {
1202 status = NVME_MD_SGL_LEN_INVALID | NVME_DNR;
1203 }
1204
1205 return status;
1206 }
1207
1208 nvme_sg_init(n, sg, nvme_addr_is_dma(n, mptr));
1209 status = nvme_map_addr(n, sg, mptr, len);
1210 if (status) {
1211 nvme_sg_unmap(sg);
1212 }
1213
1214 return status;
1215 }
1216
1217 static uint16_t nvme_map_data(NvmeCtrl *n, uint32_t nlb, NvmeRequest *req)
1218 {
1219 NvmeNamespace *ns = req->ns;
1220 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
1221 bool pi = !!NVME_ID_NS_DPS_TYPE(ns->id_ns.dps);
1222 bool pract = !!(le16_to_cpu(rw->control) & NVME_RW_PRINFO_PRACT);
1223 size_t len = nvme_l2b(ns, nlb);
1224 uint16_t status;
1225
1226 if (nvme_ns_ext(ns) &&
1227 !(pi && pract && ns->lbaf.ms == nvme_pi_tuple_size(ns))) {
1228 NvmeSg sg;
1229
1230 len += nvme_m2b(ns, nlb);
1231
1232 status = nvme_map_dptr(n, &sg, len, &req->cmd);
1233 if (status) {
1234 return status;
1235 }
1236
1237 nvme_sg_init(n, &req->sg, sg.flags & NVME_SG_DMA);
1238 nvme_sg_split(&sg, ns, &req->sg, NULL);
1239 nvme_sg_unmap(&sg);
1240
1241 return NVME_SUCCESS;
1242 }
1243
1244 return nvme_map_dptr(n, &req->sg, len, &req->cmd);
1245 }
1246
1247 static uint16_t nvme_map_mdata(NvmeCtrl *n, uint32_t nlb, NvmeRequest *req)
1248 {
1249 NvmeNamespace *ns = req->ns;
1250 size_t len = nvme_m2b(ns, nlb);
1251 uint16_t status;
1252
1253 if (nvme_ns_ext(ns)) {
1254 NvmeSg sg;
1255
1256 len += nvme_l2b(ns, nlb);
1257
1258 status = nvme_map_dptr(n, &sg, len, &req->cmd);
1259 if (status) {
1260 return status;
1261 }
1262
1263 nvme_sg_init(n, &req->sg, sg.flags & NVME_SG_DMA);
1264 nvme_sg_split(&sg, ns, NULL, &req->sg);
1265 nvme_sg_unmap(&sg);
1266
1267 return NVME_SUCCESS;
1268 }
1269
1270 return nvme_map_mptr(n, &req->sg, len, &req->cmd);
1271 }
1272
1273 static uint16_t nvme_tx_interleaved(NvmeCtrl *n, NvmeSg *sg, uint8_t *ptr,
1274 uint32_t len, uint32_t bytes,
1275 int32_t skip_bytes, int64_t offset,
1276 NvmeTxDirection dir)
1277 {
1278 hwaddr addr;
1279 uint32_t trans_len, count = bytes;
1280 bool dma = sg->flags & NVME_SG_DMA;
1281 int64_t sge_len;
1282 int sg_idx = 0;
1283 int ret;
1284
1285 assert(sg->flags & NVME_SG_ALLOC);
1286
1287 while (len) {
1288 sge_len = dma ? sg->qsg.sg[sg_idx].len : sg->iov.iov[sg_idx].iov_len;
1289
1290 if (sge_len - offset < 0) {
1291 offset -= sge_len;
1292 sg_idx++;
1293 continue;
1294 }
1295
1296 if (sge_len == offset) {
1297 offset = 0;
1298 sg_idx++;
1299 continue;
1300 }
1301
1302 trans_len = MIN(len, count);
1303 trans_len = MIN(trans_len, sge_len - offset);
1304
1305 if (dma) {
1306 addr = sg->qsg.sg[sg_idx].base + offset;
1307 } else {
1308 addr = (hwaddr)(uintptr_t)sg->iov.iov[sg_idx].iov_base + offset;
1309 }
1310
1311 if (dir == NVME_TX_DIRECTION_TO_DEVICE) {
1312 ret = nvme_addr_read(n, addr, ptr, trans_len);
1313 } else {
1314 ret = nvme_addr_write(n, addr, ptr, trans_len);
1315 }
1316
1317 if (ret) {
1318 return NVME_DATA_TRAS_ERROR;
1319 }
1320
1321 ptr += trans_len;
1322 len -= trans_len;
1323 count -= trans_len;
1324 offset += trans_len;
1325
1326 if (count == 0) {
1327 count = bytes;
1328 offset += skip_bytes;
1329 }
1330 }
1331
1332 return NVME_SUCCESS;
1333 }
1334
1335 static uint16_t nvme_tx(NvmeCtrl *n, NvmeSg *sg, void *ptr, uint32_t len,
1336 NvmeTxDirection dir)
1337 {
1338 assert(sg->flags & NVME_SG_ALLOC);
1339
1340 if (sg->flags & NVME_SG_DMA) {
1341 const MemTxAttrs attrs = MEMTXATTRS_UNSPECIFIED;
1342 dma_addr_t residual;
1343
1344 if (dir == NVME_TX_DIRECTION_TO_DEVICE) {
1345 dma_buf_write(ptr, len, &residual, &sg->qsg, attrs);
1346 } else {
1347 dma_buf_read(ptr, len, &residual, &sg->qsg, attrs);
1348 }
1349
1350 if (unlikely(residual)) {
1351 trace_pci_nvme_err_invalid_dma();
1352 return NVME_INVALID_FIELD | NVME_DNR;
1353 }
1354 } else {
1355 size_t bytes;
1356
1357 if (dir == NVME_TX_DIRECTION_TO_DEVICE) {
1358 bytes = qemu_iovec_to_buf(&sg->iov, 0, ptr, len);
1359 } else {
1360 bytes = qemu_iovec_from_buf(&sg->iov, 0, ptr, len);
1361 }
1362
1363 if (unlikely(bytes != len)) {
1364 trace_pci_nvme_err_invalid_dma();
1365 return NVME_INVALID_FIELD | NVME_DNR;
1366 }
1367 }
1368
1369 return NVME_SUCCESS;
1370 }
1371
1372 static inline uint16_t nvme_c2h(NvmeCtrl *n, void *ptr, uint32_t len,
1373 NvmeRequest *req)
1374 {
1375 uint16_t status;
1376
1377 status = nvme_map_dptr(n, &req->sg, len, &req->cmd);
1378 if (status) {
1379 return status;
1380 }
1381
1382 return nvme_tx(n, &req->sg, ptr, len, NVME_TX_DIRECTION_FROM_DEVICE);
1383 }
1384
1385 static inline uint16_t nvme_h2c(NvmeCtrl *n, void *ptr, uint32_t len,
1386 NvmeRequest *req)
1387 {
1388 uint16_t status;
1389
1390 status = nvme_map_dptr(n, &req->sg, len, &req->cmd);
1391 if (status) {
1392 return status;
1393 }
1394
1395 return nvme_tx(n, &req->sg, ptr, len, NVME_TX_DIRECTION_TO_DEVICE);
1396 }
1397
1398 uint16_t nvme_bounce_data(NvmeCtrl *n, void *ptr, uint32_t len,
1399 NvmeTxDirection dir, NvmeRequest *req)
1400 {
1401 NvmeNamespace *ns = req->ns;
1402 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
1403 bool pi = !!NVME_ID_NS_DPS_TYPE(ns->id_ns.dps);
1404 bool pract = !!(le16_to_cpu(rw->control) & NVME_RW_PRINFO_PRACT);
1405
1406 if (nvme_ns_ext(ns) &&
1407 !(pi && pract && ns->lbaf.ms == nvme_pi_tuple_size(ns))) {
1408 return nvme_tx_interleaved(n, &req->sg, ptr, len, ns->lbasz,
1409 ns->lbaf.ms, 0, dir);
1410 }
1411
1412 return nvme_tx(n, &req->sg, ptr, len, dir);
1413 }
1414
1415 uint16_t nvme_bounce_mdata(NvmeCtrl *n, void *ptr, uint32_t len,
1416 NvmeTxDirection dir, NvmeRequest *req)
1417 {
1418 NvmeNamespace *ns = req->ns;
1419 uint16_t status;
1420
1421 if (nvme_ns_ext(ns)) {
1422 return nvme_tx_interleaved(n, &req->sg, ptr, len, ns->lbaf.ms,
1423 ns->lbasz, ns->lbasz, dir);
1424 }
1425
1426 nvme_sg_unmap(&req->sg);
1427
1428 status = nvme_map_mptr(n, &req->sg, len, &req->cmd);
1429 if (status) {
1430 return status;
1431 }
1432
1433 return nvme_tx(n, &req->sg, ptr, len, dir);
1434 }
1435
1436 static inline void nvme_blk_read(BlockBackend *blk, int64_t offset,
1437 uint32_t align, BlockCompletionFunc *cb,
1438 NvmeRequest *req)
1439 {
1440 assert(req->sg.flags & NVME_SG_ALLOC);
1441
1442 if (req->sg.flags & NVME_SG_DMA) {
1443 req->aiocb = dma_blk_read(blk, &req->sg.qsg, offset, align, cb, req);
1444 } else {
1445 req->aiocb = blk_aio_preadv(blk, offset, &req->sg.iov, 0, cb, req);
1446 }
1447 }
1448
1449 static inline void nvme_blk_write(BlockBackend *blk, int64_t offset,
1450 uint32_t align, BlockCompletionFunc *cb,
1451 NvmeRequest *req)
1452 {
1453 assert(req->sg.flags & NVME_SG_ALLOC);
1454
1455 if (req->sg.flags & NVME_SG_DMA) {
1456 req->aiocb = dma_blk_write(blk, &req->sg.qsg, offset, align, cb, req);
1457 } else {
1458 req->aiocb = blk_aio_pwritev(blk, offset, &req->sg.iov, 0, cb, req);
1459 }
1460 }
1461
1462 static void nvme_update_cq_eventidx(const NvmeCQueue *cq)
1463 {
1464 uint32_t v = cpu_to_le32(cq->head);
1465
1466 trace_pci_nvme_update_cq_eventidx(cq->cqid, cq->head);
1467
1468 pci_dma_write(PCI_DEVICE(cq->ctrl), cq->ei_addr, &v, sizeof(v));
1469 }
1470
1471 static void nvme_update_cq_head(NvmeCQueue *cq)
1472 {
1473 uint32_t v;
1474
1475 pci_dma_read(PCI_DEVICE(cq->ctrl), cq->db_addr, &v, sizeof(v));
1476
1477 cq->head = le32_to_cpu(v);
1478
1479 trace_pci_nvme_update_cq_head(cq->cqid, cq->head);
1480 }
1481
1482 static void nvme_post_cqes(void *opaque)
1483 {
1484 NvmeCQueue *cq = opaque;
1485 NvmeCtrl *n = cq->ctrl;
1486 NvmeRequest *req, *next;
1487 bool pending = cq->head != cq->tail;
1488 int ret;
1489
1490 QTAILQ_FOREACH_SAFE(req, &cq->req_list, entry, next) {
1491 NvmeSQueue *sq;
1492 hwaddr addr;
1493
1494 if (n->dbbuf_enabled) {
1495 nvme_update_cq_eventidx(cq);
1496 nvme_update_cq_head(cq);
1497 }
1498
1499 if (nvme_cq_full(cq)) {
1500 break;
1501 }
1502
1503 sq = req->sq;
1504 req->cqe.status = cpu_to_le16((req->status << 1) | cq->phase);
1505 req->cqe.sq_id = cpu_to_le16(sq->sqid);
1506 req->cqe.sq_head = cpu_to_le16(sq->head);
1507 addr = cq->dma_addr + cq->tail * n->cqe_size;
1508 ret = pci_dma_write(PCI_DEVICE(n), addr, (void *)&req->cqe,
1509 sizeof(req->cqe));
1510 if (ret) {
1511 trace_pci_nvme_err_addr_write(addr);
1512 trace_pci_nvme_err_cfs();
1513 stl_le_p(&n->bar.csts, NVME_CSTS_FAILED);
1514 break;
1515 }
1516 QTAILQ_REMOVE(&cq->req_list, req, entry);
1517 nvme_inc_cq_tail(cq);
1518 nvme_sg_unmap(&req->sg);
1519 QTAILQ_INSERT_TAIL(&sq->req_list, req, entry);
1520 }
1521 if (cq->tail != cq->head) {
1522 if (cq->irq_enabled && !pending) {
1523 n->cq_pending++;
1524 }
1525
1526 nvme_irq_assert(n, cq);
1527 }
1528 }
1529
1530 static void nvme_enqueue_req_completion(NvmeCQueue *cq, NvmeRequest *req)
1531 {
1532 assert(cq->cqid == req->sq->cqid);
1533 trace_pci_nvme_enqueue_req_completion(nvme_cid(req), cq->cqid,
1534 le32_to_cpu(req->cqe.result),
1535 le32_to_cpu(req->cqe.dw1),
1536 req->status);
1537
1538 if (req->status) {
1539 trace_pci_nvme_err_req_status(nvme_cid(req), nvme_nsid(req->ns),
1540 req->status, req->cmd.opcode);
1541 }
1542
1543 QTAILQ_REMOVE(&req->sq->out_req_list, req, entry);
1544 QTAILQ_INSERT_TAIL(&cq->req_list, req, entry);
1545
1546 qemu_bh_schedule(cq->bh);
1547 }
1548
1549 static void nvme_process_aers(void *opaque)
1550 {
1551 NvmeCtrl *n = opaque;
1552 NvmeAsyncEvent *event, *next;
1553
1554 trace_pci_nvme_process_aers(n->aer_queued);
1555
1556 QTAILQ_FOREACH_SAFE(event, &n->aer_queue, entry, next) {
1557 NvmeRequest *req;
1558 NvmeAerResult *result;
1559
1560 /* can't post cqe if there is nothing to complete */
1561 if (!n->outstanding_aers) {
1562 trace_pci_nvme_no_outstanding_aers();
1563 break;
1564 }
1565
1566 /* ignore if masked (cqe posted, but event not cleared) */
1567 if (n->aer_mask & (1 << event->result.event_type)) {
1568 trace_pci_nvme_aer_masked(event->result.event_type, n->aer_mask);
1569 continue;
1570 }
1571
1572 QTAILQ_REMOVE(&n->aer_queue, event, entry);
1573 n->aer_queued--;
1574
1575 n->aer_mask |= 1 << event->result.event_type;
1576 n->outstanding_aers--;
1577
1578 req = n->aer_reqs[n->outstanding_aers];
1579
1580 result = (NvmeAerResult *) &req->cqe.result;
1581 result->event_type = event->result.event_type;
1582 result->event_info = event->result.event_info;
1583 result->log_page = event->result.log_page;
1584 g_free(event);
1585
1586 trace_pci_nvme_aer_post_cqe(result->event_type, result->event_info,
1587 result->log_page);
1588
1589 nvme_enqueue_req_completion(&n->admin_cq, req);
1590 }
1591 }
1592
1593 static void nvme_enqueue_event(NvmeCtrl *n, uint8_t event_type,
1594 uint8_t event_info, uint8_t log_page)
1595 {
1596 NvmeAsyncEvent *event;
1597
1598 trace_pci_nvme_enqueue_event(event_type, event_info, log_page);
1599
1600 if (n->aer_queued == n->params.aer_max_queued) {
1601 trace_pci_nvme_enqueue_event_noqueue(n->aer_queued);
1602 return;
1603 }
1604
1605 event = g_new(NvmeAsyncEvent, 1);
1606 event->result = (NvmeAerResult) {
1607 .event_type = event_type,
1608 .event_info = event_info,
1609 .log_page = log_page,
1610 };
1611
1612 QTAILQ_INSERT_TAIL(&n->aer_queue, event, entry);
1613 n->aer_queued++;
1614
1615 nvme_process_aers(n);
1616 }
1617
1618 static void nvme_smart_event(NvmeCtrl *n, uint8_t event)
1619 {
1620 uint8_t aer_info;
1621
1622 /* Ref SPEC <Asynchronous Event Information 0x2013 SMART / Health Status> */
1623 if (!(NVME_AEC_SMART(n->features.async_config) & event)) {
1624 return;
1625 }
1626
1627 switch (event) {
1628 case NVME_SMART_SPARE:
1629 aer_info = NVME_AER_INFO_SMART_SPARE_THRESH;
1630 break;
1631 case NVME_SMART_TEMPERATURE:
1632 aer_info = NVME_AER_INFO_SMART_TEMP_THRESH;
1633 break;
1634 case NVME_SMART_RELIABILITY:
1635 case NVME_SMART_MEDIA_READ_ONLY:
1636 case NVME_SMART_FAILED_VOLATILE_MEDIA:
1637 case NVME_SMART_PMR_UNRELIABLE:
1638 aer_info = NVME_AER_INFO_SMART_RELIABILITY;
1639 break;
1640 default:
1641 return;
1642 }
1643
1644 nvme_enqueue_event(n, NVME_AER_TYPE_SMART, aer_info, NVME_LOG_SMART_INFO);
1645 }
1646
1647 static void nvme_clear_events(NvmeCtrl *n, uint8_t event_type)
1648 {
1649 n->aer_mask &= ~(1 << event_type);
1650 if (!QTAILQ_EMPTY(&n->aer_queue)) {
1651 nvme_process_aers(n);
1652 }
1653 }
1654
1655 static inline uint16_t nvme_check_mdts(NvmeCtrl *n, size_t len)
1656 {
1657 uint8_t mdts = n->params.mdts;
1658
1659 if (mdts && len > n->page_size << mdts) {
1660 trace_pci_nvme_err_mdts(len);
1661 return NVME_INVALID_FIELD | NVME_DNR;
1662 }
1663
1664 return NVME_SUCCESS;
1665 }
1666
1667 static inline uint16_t nvme_check_bounds(NvmeNamespace *ns, uint64_t slba,
1668 uint32_t nlb)
1669 {
1670 uint64_t nsze = le64_to_cpu(ns->id_ns.nsze);
1671
1672 if (unlikely(UINT64_MAX - slba < nlb || slba + nlb > nsze)) {
1673 trace_pci_nvme_err_invalid_lba_range(slba, nlb, nsze);
1674 return NVME_LBA_RANGE | NVME_DNR;
1675 }
1676
1677 return NVME_SUCCESS;
1678 }
1679
1680 static int nvme_block_status_all(NvmeNamespace *ns, uint64_t slba,
1681 uint32_t nlb, int flags)
1682 {
1683 BlockDriverState *bs = blk_bs(ns->blkconf.blk);
1684
1685 int64_t pnum = 0, bytes = nvme_l2b(ns, nlb);
1686 int64_t offset = nvme_l2b(ns, slba);
1687 int ret;
1688
1689 /*
1690 * `pnum` holds the number of bytes after offset that shares the same
1691 * allocation status as the byte at offset. If `pnum` is different from
1692 * `bytes`, we should check the allocation status of the next range and
1693 * continue this until all bytes have been checked.
1694 */
1695 do {
1696 bytes -= pnum;
1697
1698 ret = bdrv_block_status(bs, offset, bytes, &pnum, NULL, NULL);
1699 if (ret < 0) {
1700 return ret;
1701 }
1702
1703
1704 trace_pci_nvme_block_status(offset, bytes, pnum, ret,
1705 !!(ret & BDRV_BLOCK_ZERO));
1706
1707 if (!(ret & flags)) {
1708 return 1;
1709 }
1710
1711 offset += pnum;
1712 } while (pnum != bytes);
1713
1714 return 0;
1715 }
1716
1717 static uint16_t nvme_check_dulbe(NvmeNamespace *ns, uint64_t slba,
1718 uint32_t nlb)
1719 {
1720 int ret;
1721 Error *err = NULL;
1722
1723 ret = nvme_block_status_all(ns, slba, nlb, BDRV_BLOCK_DATA);
1724 if (ret) {
1725 if (ret < 0) {
1726 error_setg_errno(&err, -ret, "unable to get block status");
1727 error_report_err(err);
1728
1729 return NVME_INTERNAL_DEV_ERROR;
1730 }
1731
1732 return NVME_DULB;
1733 }
1734
1735 return NVME_SUCCESS;
1736 }
1737
1738 static void nvme_aio_err(NvmeRequest *req, int ret)
1739 {
1740 uint16_t status = NVME_SUCCESS;
1741 Error *local_err = NULL;
1742
1743 switch (req->cmd.opcode) {
1744 case NVME_CMD_READ:
1745 status = NVME_UNRECOVERED_READ;
1746 break;
1747 case NVME_CMD_FLUSH:
1748 case NVME_CMD_WRITE:
1749 case NVME_CMD_WRITE_ZEROES:
1750 case NVME_CMD_ZONE_APPEND:
1751 status = NVME_WRITE_FAULT;
1752 break;
1753 default:
1754 status = NVME_INTERNAL_DEV_ERROR;
1755 break;
1756 }
1757
1758 trace_pci_nvme_err_aio(nvme_cid(req), strerror(-ret), status);
1759
1760 error_setg_errno(&local_err, -ret, "aio failed");
1761 error_report_err(local_err);
1762
1763 /*
1764 * Set the command status code to the first encountered error but allow a
1765 * subsequent Internal Device Error to trump it.
1766 */
1767 if (req->status && status != NVME_INTERNAL_DEV_ERROR) {
1768 return;
1769 }
1770
1771 req->status = status;
1772 }
1773
1774 static inline uint32_t nvme_zone_idx(NvmeNamespace *ns, uint64_t slba)
1775 {
1776 return ns->zone_size_log2 > 0 ? slba >> ns->zone_size_log2 :
1777 slba / ns->zone_size;
1778 }
1779
1780 static inline NvmeZone *nvme_get_zone_by_slba(NvmeNamespace *ns, uint64_t slba)
1781 {
1782 uint32_t zone_idx = nvme_zone_idx(ns, slba);
1783
1784 if (zone_idx >= ns->num_zones) {
1785 return NULL;
1786 }
1787
1788 return &ns->zone_array[zone_idx];
1789 }
1790
1791 static uint16_t nvme_check_zone_state_for_write(NvmeZone *zone)
1792 {
1793 uint64_t zslba = zone->d.zslba;
1794
1795 switch (nvme_get_zone_state(zone)) {
1796 case NVME_ZONE_STATE_EMPTY:
1797 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1798 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1799 case NVME_ZONE_STATE_CLOSED:
1800 return NVME_SUCCESS;
1801 case NVME_ZONE_STATE_FULL:
1802 trace_pci_nvme_err_zone_is_full(zslba);
1803 return NVME_ZONE_FULL;
1804 case NVME_ZONE_STATE_OFFLINE:
1805 trace_pci_nvme_err_zone_is_offline(zslba);
1806 return NVME_ZONE_OFFLINE;
1807 case NVME_ZONE_STATE_READ_ONLY:
1808 trace_pci_nvme_err_zone_is_read_only(zslba);
1809 return NVME_ZONE_READ_ONLY;
1810 default:
1811 assert(false);
1812 }
1813
1814 return NVME_INTERNAL_DEV_ERROR;
1815 }
1816
1817 static uint16_t nvme_check_zone_write(NvmeNamespace *ns, NvmeZone *zone,
1818 uint64_t slba, uint32_t nlb)
1819 {
1820 uint64_t zcap = nvme_zone_wr_boundary(zone);
1821 uint16_t status;
1822
1823 status = nvme_check_zone_state_for_write(zone);
1824 if (status) {
1825 return status;
1826 }
1827
1828 if (zone->d.za & NVME_ZA_ZRWA_VALID) {
1829 uint64_t ezrwa = zone->w_ptr + 2 * ns->zns.zrwas;
1830
1831 if (slba < zone->w_ptr || slba + nlb > ezrwa) {
1832 trace_pci_nvme_err_zone_invalid_write(slba, zone->w_ptr);
1833 return NVME_ZONE_INVALID_WRITE;
1834 }
1835 } else {
1836 if (unlikely(slba != zone->w_ptr)) {
1837 trace_pci_nvme_err_write_not_at_wp(slba, zone->d.zslba,
1838 zone->w_ptr);
1839 return NVME_ZONE_INVALID_WRITE;
1840 }
1841 }
1842
1843 if (unlikely((slba + nlb) > zcap)) {
1844 trace_pci_nvme_err_zone_boundary(slba, nlb, zcap);
1845 return NVME_ZONE_BOUNDARY_ERROR;
1846 }
1847
1848 return NVME_SUCCESS;
1849 }
1850
1851 static uint16_t nvme_check_zone_state_for_read(NvmeZone *zone)
1852 {
1853 switch (nvme_get_zone_state(zone)) {
1854 case NVME_ZONE_STATE_EMPTY:
1855 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1856 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1857 case NVME_ZONE_STATE_FULL:
1858 case NVME_ZONE_STATE_CLOSED:
1859 case NVME_ZONE_STATE_READ_ONLY:
1860 return NVME_SUCCESS;
1861 case NVME_ZONE_STATE_OFFLINE:
1862 trace_pci_nvme_err_zone_is_offline(zone->d.zslba);
1863 return NVME_ZONE_OFFLINE;
1864 default:
1865 assert(false);
1866 }
1867
1868 return NVME_INTERNAL_DEV_ERROR;
1869 }
1870
1871 static uint16_t nvme_check_zone_read(NvmeNamespace *ns, uint64_t slba,
1872 uint32_t nlb)
1873 {
1874 NvmeZone *zone;
1875 uint64_t bndry, end;
1876 uint16_t status;
1877
1878 zone = nvme_get_zone_by_slba(ns, slba);
1879 assert(zone);
1880
1881 bndry = nvme_zone_rd_boundary(ns, zone);
1882 end = slba + nlb;
1883
1884 status = nvme_check_zone_state_for_read(zone);
1885 if (status) {
1886 ;
1887 } else if (unlikely(end > bndry)) {
1888 if (!ns->params.cross_zone_read) {
1889 status = NVME_ZONE_BOUNDARY_ERROR;
1890 } else {
1891 /*
1892 * Read across zone boundary - check that all subsequent
1893 * zones that are being read have an appropriate state.
1894 */
1895 do {
1896 zone++;
1897 status = nvme_check_zone_state_for_read(zone);
1898 if (status) {
1899 break;
1900 }
1901 } while (end > nvme_zone_rd_boundary(ns, zone));
1902 }
1903 }
1904
1905 return status;
1906 }
1907
1908 static uint16_t nvme_zrm_finish(NvmeNamespace *ns, NvmeZone *zone)
1909 {
1910 switch (nvme_get_zone_state(zone)) {
1911 case NVME_ZONE_STATE_FULL:
1912 return NVME_SUCCESS;
1913
1914 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1915 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1916 nvme_aor_dec_open(ns);
1917 /* fallthrough */
1918 case NVME_ZONE_STATE_CLOSED:
1919 nvme_aor_dec_active(ns);
1920
1921 if (zone->d.za & NVME_ZA_ZRWA_VALID) {
1922 zone->d.za &= ~NVME_ZA_ZRWA_VALID;
1923 if (ns->params.numzrwa) {
1924 ns->zns.numzrwa++;
1925 }
1926 }
1927
1928 /* fallthrough */
1929 case NVME_ZONE_STATE_EMPTY:
1930 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_FULL);
1931 return NVME_SUCCESS;
1932
1933 default:
1934 return NVME_ZONE_INVAL_TRANSITION;
1935 }
1936 }
1937
1938 static uint16_t nvme_zrm_close(NvmeNamespace *ns, NvmeZone *zone)
1939 {
1940 switch (nvme_get_zone_state(zone)) {
1941 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1942 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1943 nvme_aor_dec_open(ns);
1944 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_CLOSED);
1945 /* fall through */
1946 case NVME_ZONE_STATE_CLOSED:
1947 return NVME_SUCCESS;
1948
1949 default:
1950 return NVME_ZONE_INVAL_TRANSITION;
1951 }
1952 }
1953
1954 static uint16_t nvme_zrm_reset(NvmeNamespace *ns, NvmeZone *zone)
1955 {
1956 switch (nvme_get_zone_state(zone)) {
1957 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1958 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1959 nvme_aor_dec_open(ns);
1960 /* fallthrough */
1961 case NVME_ZONE_STATE_CLOSED:
1962 nvme_aor_dec_active(ns);
1963
1964 if (zone->d.za & NVME_ZA_ZRWA_VALID) {
1965 if (ns->params.numzrwa) {
1966 ns->zns.numzrwa++;
1967 }
1968 }
1969
1970 /* fallthrough */
1971 case NVME_ZONE_STATE_FULL:
1972 zone->w_ptr = zone->d.zslba;
1973 zone->d.wp = zone->w_ptr;
1974 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_EMPTY);
1975 /* fallthrough */
1976 case NVME_ZONE_STATE_EMPTY:
1977 return NVME_SUCCESS;
1978
1979 default:
1980 return NVME_ZONE_INVAL_TRANSITION;
1981 }
1982 }
1983
1984 static void nvme_zrm_auto_transition_zone(NvmeNamespace *ns)
1985 {
1986 NvmeZone *zone;
1987
1988 if (ns->params.max_open_zones &&
1989 ns->nr_open_zones == ns->params.max_open_zones) {
1990 zone = QTAILQ_FIRST(&ns->imp_open_zones);
1991 if (zone) {
1992 /*
1993 * Automatically close this implicitly open zone.
1994 */
1995 QTAILQ_REMOVE(&ns->imp_open_zones, zone, entry);
1996 nvme_zrm_close(ns, zone);
1997 }
1998 }
1999 }
2000
2001 enum {
2002 NVME_ZRM_AUTO = 1 << 0,
2003 NVME_ZRM_ZRWA = 1 << 1,
2004 };
2005
2006 static uint16_t nvme_zrm_open_flags(NvmeCtrl *n, NvmeNamespace *ns,
2007 NvmeZone *zone, int flags)
2008 {
2009 int act = 0;
2010 uint16_t status;
2011
2012 switch (nvme_get_zone_state(zone)) {
2013 case NVME_ZONE_STATE_EMPTY:
2014 act = 1;
2015
2016 /* fallthrough */
2017
2018 case NVME_ZONE_STATE_CLOSED:
2019 if (n->params.auto_transition_zones) {
2020 nvme_zrm_auto_transition_zone(ns);
2021 }
2022 status = nvme_zns_check_resources(ns, act, 1,
2023 (flags & NVME_ZRM_ZRWA) ? 1 : 0);
2024 if (status) {
2025 return status;
2026 }
2027
2028 if (act) {
2029 nvme_aor_inc_active(ns);
2030 }
2031
2032 nvme_aor_inc_open(ns);
2033
2034 if (flags & NVME_ZRM_AUTO) {
2035 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_IMPLICITLY_OPEN);
2036 return NVME_SUCCESS;
2037 }
2038
2039 /* fallthrough */
2040
2041 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
2042 if (flags & NVME_ZRM_AUTO) {
2043 return NVME_SUCCESS;
2044 }
2045
2046 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_EXPLICITLY_OPEN);
2047
2048 /* fallthrough */
2049
2050 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
2051 if (flags & NVME_ZRM_ZRWA) {
2052 ns->zns.numzrwa--;
2053
2054 zone->d.za |= NVME_ZA_ZRWA_VALID;
2055 }
2056
2057 return NVME_SUCCESS;
2058
2059 default:
2060 return NVME_ZONE_INVAL_TRANSITION;
2061 }
2062 }
2063
2064 static inline uint16_t nvme_zrm_auto(NvmeCtrl *n, NvmeNamespace *ns,
2065 NvmeZone *zone)
2066 {
2067 return nvme_zrm_open_flags(n, ns, zone, NVME_ZRM_AUTO);
2068 }
2069
2070 static void nvme_advance_zone_wp(NvmeNamespace *ns, NvmeZone *zone,
2071 uint32_t nlb)
2072 {
2073 zone->d.wp += nlb;
2074
2075 if (zone->d.wp == nvme_zone_wr_boundary(zone)) {
2076 nvme_zrm_finish(ns, zone);
2077 }
2078 }
2079
2080 static void nvme_zoned_zrwa_implicit_flush(NvmeNamespace *ns, NvmeZone *zone,
2081 uint32_t nlbc)
2082 {
2083 uint16_t nzrwafgs = DIV_ROUND_UP(nlbc, ns->zns.zrwafg);
2084
2085 nlbc = nzrwafgs * ns->zns.zrwafg;
2086
2087 trace_pci_nvme_zoned_zrwa_implicit_flush(zone->d.zslba, nlbc);
2088
2089 zone->w_ptr += nlbc;
2090
2091 nvme_advance_zone_wp(ns, zone, nlbc);
2092 }
2093
2094 static void nvme_finalize_zoned_write(NvmeNamespace *ns, NvmeRequest *req)
2095 {
2096 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2097 NvmeZone *zone;
2098 uint64_t slba;
2099 uint32_t nlb;
2100
2101 slba = le64_to_cpu(rw->slba);
2102 nlb = le16_to_cpu(rw->nlb) + 1;
2103 zone = nvme_get_zone_by_slba(ns, slba);
2104 assert(zone);
2105
2106 if (zone->d.za & NVME_ZA_ZRWA_VALID) {
2107 uint64_t ezrwa = zone->w_ptr + ns->zns.zrwas - 1;
2108 uint64_t elba = slba + nlb - 1;
2109
2110 if (elba > ezrwa) {
2111 nvme_zoned_zrwa_implicit_flush(ns, zone, elba - ezrwa);
2112 }
2113
2114 return;
2115 }
2116
2117 nvme_advance_zone_wp(ns, zone, nlb);
2118 }
2119
2120 static inline bool nvme_is_write(NvmeRequest *req)
2121 {
2122 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2123
2124 return rw->opcode == NVME_CMD_WRITE ||
2125 rw->opcode == NVME_CMD_ZONE_APPEND ||
2126 rw->opcode == NVME_CMD_WRITE_ZEROES;
2127 }
2128
2129 static AioContext *nvme_get_aio_context(BlockAIOCB *acb)
2130 {
2131 return qemu_get_aio_context();
2132 }
2133
2134 static void nvme_misc_cb(void *opaque, int ret)
2135 {
2136 NvmeRequest *req = opaque;
2137
2138 trace_pci_nvme_misc_cb(nvme_cid(req));
2139
2140 if (ret) {
2141 nvme_aio_err(req, ret);
2142 }
2143
2144 nvme_enqueue_req_completion(nvme_cq(req), req);
2145 }
2146
2147 void nvme_rw_complete_cb(void *opaque, int ret)
2148 {
2149 NvmeRequest *req = opaque;
2150 NvmeNamespace *ns = req->ns;
2151 BlockBackend *blk = ns->blkconf.blk;
2152 BlockAcctCookie *acct = &req->acct;
2153 BlockAcctStats *stats = blk_get_stats(blk);
2154
2155 trace_pci_nvme_rw_complete_cb(nvme_cid(req), blk_name(blk));
2156
2157 if (ret) {
2158 block_acct_failed(stats, acct);
2159 nvme_aio_err(req, ret);
2160 } else {
2161 block_acct_done(stats, acct);
2162 }
2163
2164 if (ns->params.zoned && nvme_is_write(req)) {
2165 nvme_finalize_zoned_write(ns, req);
2166 }
2167
2168 nvme_enqueue_req_completion(nvme_cq(req), req);
2169 }
2170
2171 static void nvme_rw_cb(void *opaque, int ret)
2172 {
2173 NvmeRequest *req = opaque;
2174 NvmeNamespace *ns = req->ns;
2175
2176 BlockBackend *blk = ns->blkconf.blk;
2177
2178 trace_pci_nvme_rw_cb(nvme_cid(req), blk_name(blk));
2179
2180 if (ret) {
2181 goto out;
2182 }
2183
2184 if (ns->lbaf.ms) {
2185 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2186 uint64_t slba = le64_to_cpu(rw->slba);
2187 uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb) + 1;
2188 uint64_t offset = nvme_moff(ns, slba);
2189
2190 if (req->cmd.opcode == NVME_CMD_WRITE_ZEROES) {
2191 size_t mlen = nvme_m2b(ns, nlb);
2192
2193 req->aiocb = blk_aio_pwrite_zeroes(blk, offset, mlen,
2194 BDRV_REQ_MAY_UNMAP,
2195 nvme_rw_complete_cb, req);
2196 return;
2197 }
2198
2199 if (nvme_ns_ext(ns) || req->cmd.mptr) {
2200 uint16_t status;
2201
2202 nvme_sg_unmap(&req->sg);
2203 status = nvme_map_mdata(nvme_ctrl(req), nlb, req);
2204 if (status) {
2205 ret = -EFAULT;
2206 goto out;
2207 }
2208
2209 if (req->cmd.opcode == NVME_CMD_READ) {
2210 return nvme_blk_read(blk, offset, 1, nvme_rw_complete_cb, req);
2211 }
2212
2213 return nvme_blk_write(blk, offset, 1, nvme_rw_complete_cb, req);
2214 }
2215 }
2216
2217 out:
2218 nvme_rw_complete_cb(req, ret);
2219 }
2220
2221 static void nvme_verify_cb(void *opaque, int ret)
2222 {
2223 NvmeBounceContext *ctx = opaque;
2224 NvmeRequest *req = ctx->req;
2225 NvmeNamespace *ns = req->ns;
2226 BlockBackend *blk = ns->blkconf.blk;
2227 BlockAcctCookie *acct = &req->acct;
2228 BlockAcctStats *stats = blk_get_stats(blk);
2229 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2230 uint64_t slba = le64_to_cpu(rw->slba);
2231 uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
2232 uint16_t apptag = le16_to_cpu(rw->apptag);
2233 uint16_t appmask = le16_to_cpu(rw->appmask);
2234 uint64_t reftag = le32_to_cpu(rw->reftag);
2235 uint64_t cdw3 = le32_to_cpu(rw->cdw3);
2236 uint16_t status;
2237
2238 reftag |= cdw3 << 32;
2239
2240 trace_pci_nvme_verify_cb(nvme_cid(req), prinfo, apptag, appmask, reftag);
2241
2242 if (ret) {
2243 block_acct_failed(stats, acct);
2244 nvme_aio_err(req, ret);
2245 goto out;
2246 }
2247
2248 block_acct_done(stats, acct);
2249
2250 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
2251 status = nvme_dif_mangle_mdata(ns, ctx->mdata.bounce,
2252 ctx->mdata.iov.size, slba);
2253 if (status) {
2254 req->status = status;
2255 goto out;
2256 }
2257
2258 req->status = nvme_dif_check(ns, ctx->data.bounce, ctx->data.iov.size,
2259 ctx->mdata.bounce, ctx->mdata.iov.size,
2260 prinfo, slba, apptag, appmask, &reftag);
2261 }
2262
2263 out:
2264 qemu_iovec_destroy(&ctx->data.iov);
2265 g_free(ctx->data.bounce);
2266
2267 qemu_iovec_destroy(&ctx->mdata.iov);
2268 g_free(ctx->mdata.bounce);
2269
2270 g_free(ctx);
2271
2272 nvme_enqueue_req_completion(nvme_cq(req), req);
2273 }
2274
2275
2276 static void nvme_verify_mdata_in_cb(void *opaque, int ret)
2277 {
2278 NvmeBounceContext *ctx = opaque;
2279 NvmeRequest *req = ctx->req;
2280 NvmeNamespace *ns = req->ns;
2281 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2282 uint64_t slba = le64_to_cpu(rw->slba);
2283 uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
2284 size_t mlen = nvme_m2b(ns, nlb);
2285 uint64_t offset = nvme_moff(ns, slba);
2286 BlockBackend *blk = ns->blkconf.blk;
2287
2288 trace_pci_nvme_verify_mdata_in_cb(nvme_cid(req), blk_name(blk));
2289
2290 if (ret) {
2291 goto out;
2292 }
2293
2294 ctx->mdata.bounce = g_malloc(mlen);
2295
2296 qemu_iovec_reset(&ctx->mdata.iov);
2297 qemu_iovec_add(&ctx->mdata.iov, ctx->mdata.bounce, mlen);
2298
2299 req->aiocb = blk_aio_preadv(blk, offset, &ctx->mdata.iov, 0,
2300 nvme_verify_cb, ctx);
2301 return;
2302
2303 out:
2304 nvme_verify_cb(ctx, ret);
2305 }
2306
2307 struct nvme_compare_ctx {
2308 struct {
2309 QEMUIOVector iov;
2310 uint8_t *bounce;
2311 } data;
2312
2313 struct {
2314 QEMUIOVector iov;
2315 uint8_t *bounce;
2316 } mdata;
2317 };
2318
2319 static void nvme_compare_mdata_cb(void *opaque, int ret)
2320 {
2321 NvmeRequest *req = opaque;
2322 NvmeNamespace *ns = req->ns;
2323 NvmeCtrl *n = nvme_ctrl(req);
2324 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2325 uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
2326 uint16_t apptag = le16_to_cpu(rw->apptag);
2327 uint16_t appmask = le16_to_cpu(rw->appmask);
2328 uint64_t reftag = le32_to_cpu(rw->reftag);
2329 uint64_t cdw3 = le32_to_cpu(rw->cdw3);
2330 struct nvme_compare_ctx *ctx = req->opaque;
2331 g_autofree uint8_t *buf = NULL;
2332 BlockBackend *blk = ns->blkconf.blk;
2333 BlockAcctCookie *acct = &req->acct;
2334 BlockAcctStats *stats = blk_get_stats(blk);
2335 uint16_t status = NVME_SUCCESS;
2336
2337 reftag |= cdw3 << 32;
2338
2339 trace_pci_nvme_compare_mdata_cb(nvme_cid(req));
2340
2341 if (ret) {
2342 block_acct_failed(stats, acct);
2343 nvme_aio_err(req, ret);
2344 goto out;
2345 }
2346
2347 buf = g_malloc(ctx->mdata.iov.size);
2348
2349 status = nvme_bounce_mdata(n, buf, ctx->mdata.iov.size,
2350 NVME_TX_DIRECTION_TO_DEVICE, req);
2351 if (status) {
2352 req->status = status;
2353 goto out;
2354 }
2355
2356 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
2357 uint64_t slba = le64_to_cpu(rw->slba);
2358 uint8_t *bufp;
2359 uint8_t *mbufp = ctx->mdata.bounce;
2360 uint8_t *end = mbufp + ctx->mdata.iov.size;
2361 int16_t pil = 0;
2362
2363 status = nvme_dif_check(ns, ctx->data.bounce, ctx->data.iov.size,
2364 ctx->mdata.bounce, ctx->mdata.iov.size, prinfo,
2365 slba, apptag, appmask, &reftag);
2366 if (status) {
2367 req->status = status;
2368 goto out;
2369 }
2370
2371 /*
2372 * When formatted with protection information, do not compare the DIF
2373 * tuple.
2374 */
2375 if (!(ns->id_ns.dps & NVME_ID_NS_DPS_FIRST_EIGHT)) {
2376 pil = ns->lbaf.ms - nvme_pi_tuple_size(ns);
2377 }
2378
2379 for (bufp = buf; mbufp < end; bufp += ns->lbaf.ms, mbufp += ns->lbaf.ms) {
2380 if (memcmp(bufp + pil, mbufp + pil, ns->lbaf.ms - pil)) {
2381 req->status = NVME_CMP_FAILURE | NVME_DNR;
2382 goto out;
2383 }
2384 }
2385
2386 goto out;
2387 }
2388
2389 if (memcmp(buf, ctx->mdata.bounce, ctx->mdata.iov.size)) {
2390 req->status = NVME_CMP_FAILURE | NVME_DNR;
2391 goto out;
2392 }
2393
2394 block_acct_done(stats, acct);
2395
2396 out:
2397 qemu_iovec_destroy(&ctx->data.iov);
2398 g_free(ctx->data.bounce);
2399
2400 qemu_iovec_destroy(&ctx->mdata.iov);
2401 g_free(ctx->mdata.bounce);
2402
2403 g_free(ctx);
2404
2405 nvme_enqueue_req_completion(nvme_cq(req), req);
2406 }
2407
2408 static void nvme_compare_data_cb(void *opaque, int ret)
2409 {
2410 NvmeRequest *req = opaque;
2411 NvmeCtrl *n = nvme_ctrl(req);
2412 NvmeNamespace *ns = req->ns;
2413 BlockBackend *blk = ns->blkconf.blk;
2414 BlockAcctCookie *acct = &req->acct;
2415 BlockAcctStats *stats = blk_get_stats(blk);
2416
2417 struct nvme_compare_ctx *ctx = req->opaque;
2418 g_autofree uint8_t *buf = NULL;
2419 uint16_t status;
2420
2421 trace_pci_nvme_compare_data_cb(nvme_cid(req));
2422
2423 if (ret) {
2424 block_acct_failed(stats, acct);
2425 nvme_aio_err(req, ret);
2426 goto out;
2427 }
2428
2429 buf = g_malloc(ctx->data.iov.size);
2430
2431 status = nvme_bounce_data(n, buf, ctx->data.iov.size,
2432 NVME_TX_DIRECTION_TO_DEVICE, req);
2433 if (status) {
2434 req->status = status;
2435 goto out;
2436 }
2437
2438 if (memcmp(buf, ctx->data.bounce, ctx->data.iov.size)) {
2439 req->status = NVME_CMP_FAILURE | NVME_DNR;
2440 goto out;
2441 }
2442
2443 if (ns->lbaf.ms) {
2444 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2445 uint64_t slba = le64_to_cpu(rw->slba);
2446 uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
2447 size_t mlen = nvme_m2b(ns, nlb);
2448 uint64_t offset = nvme_moff(ns, slba);
2449
2450 ctx->mdata.bounce = g_malloc(mlen);
2451
2452 qemu_iovec_init(&ctx->mdata.iov, 1);
2453 qemu_iovec_add(&ctx->mdata.iov, ctx->mdata.bounce, mlen);
2454
2455 req->aiocb = blk_aio_preadv(blk, offset, &ctx->mdata.iov, 0,
2456 nvme_compare_mdata_cb, req);
2457 return;
2458 }
2459
2460 block_acct_done(stats, acct);
2461
2462 out:
2463 qemu_iovec_destroy(&ctx->data.iov);
2464 g_free(ctx->data.bounce);
2465 g_free(ctx);
2466
2467 nvme_enqueue_req_completion(nvme_cq(req), req);
2468 }
2469
2470 typedef struct NvmeDSMAIOCB {
2471 BlockAIOCB common;
2472 BlockAIOCB *aiocb;
2473 NvmeRequest *req;
2474 int ret;
2475
2476 NvmeDsmRange *range;
2477 unsigned int nr;
2478 unsigned int idx;
2479 } NvmeDSMAIOCB;
2480
2481 static void nvme_dsm_cancel(BlockAIOCB *aiocb)
2482 {
2483 NvmeDSMAIOCB *iocb = container_of(aiocb, NvmeDSMAIOCB, common);
2484
2485 /* break nvme_dsm_cb loop */
2486 iocb->idx = iocb->nr;
2487 iocb->ret = -ECANCELED;
2488
2489 if (iocb->aiocb) {
2490 blk_aio_cancel_async(iocb->aiocb);
2491 iocb->aiocb = NULL;
2492 } else {
2493 /*
2494 * We only reach this if nvme_dsm_cancel() has already been called or
2495 * the command ran to completion.
2496 */
2497 assert(iocb->idx == iocb->nr);
2498 }
2499 }
2500
2501 static const AIOCBInfo nvme_dsm_aiocb_info = {
2502 .aiocb_size = sizeof(NvmeDSMAIOCB),
2503 .cancel_async = nvme_dsm_cancel,
2504 };
2505
2506 static void nvme_dsm_cb(void *opaque, int ret);
2507
2508 static void nvme_dsm_md_cb(void *opaque, int ret)
2509 {
2510 NvmeDSMAIOCB *iocb = opaque;
2511 NvmeRequest *req = iocb->req;
2512 NvmeNamespace *ns = req->ns;
2513 NvmeDsmRange *range;
2514 uint64_t slba;
2515 uint32_t nlb;
2516
2517 if (ret < 0 || iocb->ret < 0 || !ns->lbaf.ms) {
2518 goto done;
2519 }
2520
2521 range = &iocb->range[iocb->idx - 1];
2522 slba = le64_to_cpu(range->slba);
2523 nlb = le32_to_cpu(range->nlb);
2524
2525 /*
2526 * Check that all block were discarded (zeroed); otherwise we do not zero
2527 * the metadata.
2528 */
2529
2530 ret = nvme_block_status_all(ns, slba, nlb, BDRV_BLOCK_ZERO);
2531 if (ret) {
2532 if (ret < 0) {
2533 goto done;
2534 }
2535
2536 nvme_dsm_cb(iocb, 0);
2537 return;
2538 }
2539
2540 iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk, nvme_moff(ns, slba),
2541 nvme_m2b(ns, nlb), BDRV_REQ_MAY_UNMAP,
2542 nvme_dsm_cb, iocb);
2543 return;
2544
2545 done:
2546 nvme_dsm_cb(iocb, ret);
2547 }
2548
2549 static void nvme_dsm_cb(void *opaque, int ret)
2550 {
2551 NvmeDSMAIOCB *iocb = opaque;
2552 NvmeRequest *req = iocb->req;
2553 NvmeCtrl *n = nvme_ctrl(req);
2554 NvmeNamespace *ns = req->ns;
2555 NvmeDsmRange *range;
2556 uint64_t slba;
2557 uint32_t nlb;
2558
2559 if (iocb->ret < 0) {
2560 goto done;
2561 } else if (ret < 0) {
2562 iocb->ret = ret;
2563 goto done;
2564 }
2565
2566 next:
2567 if (iocb->idx == iocb->nr) {
2568 goto done;
2569 }
2570
2571 range = &iocb->range[iocb->idx++];
2572 slba = le64_to_cpu(range->slba);
2573 nlb = le32_to_cpu(range->nlb);
2574
2575 trace_pci_nvme_dsm_deallocate(slba, nlb);
2576
2577 if (nlb > n->dmrsl) {
2578 trace_pci_nvme_dsm_single_range_limit_exceeded(nlb, n->dmrsl);
2579 goto next;
2580 }
2581
2582 if (nvme_check_bounds(ns, slba, nlb)) {
2583 trace_pci_nvme_err_invalid_lba_range(slba, nlb,
2584 ns->id_ns.nsze);
2585 goto next;
2586 }
2587
2588 iocb->aiocb = blk_aio_pdiscard(ns->blkconf.blk, nvme_l2b(ns, slba),
2589 nvme_l2b(ns, nlb),
2590 nvme_dsm_md_cb, iocb);
2591 return;
2592
2593 done:
2594 iocb->aiocb = NULL;
2595 iocb->common.cb(iocb->common.opaque, iocb->ret);
2596 qemu_aio_unref(iocb);
2597 }
2598
2599 static uint16_t nvme_dsm(NvmeCtrl *n, NvmeRequest *req)
2600 {
2601 NvmeNamespace *ns = req->ns;
2602 NvmeDsmCmd *dsm = (NvmeDsmCmd *) &req->cmd;
2603 uint32_t attr = le32_to_cpu(dsm->attributes);
2604 uint32_t nr = (le32_to_cpu(dsm->nr) & 0xff) + 1;
2605 uint16_t status = NVME_SUCCESS;
2606
2607 trace_pci_nvme_dsm(nr, attr);
2608
2609 if (attr & NVME_DSMGMT_AD) {
2610 NvmeDSMAIOCB *iocb = blk_aio_get(&nvme_dsm_aiocb_info, ns->blkconf.blk,
2611 nvme_misc_cb, req);
2612
2613 iocb->req = req;
2614 iocb->ret = 0;
2615 iocb->range = g_new(NvmeDsmRange, nr);
2616 iocb->nr = nr;
2617 iocb->idx = 0;
2618
2619 status = nvme_h2c(n, (uint8_t *)iocb->range, sizeof(NvmeDsmRange) * nr,
2620 req);
2621 if (status) {
2622 g_free(iocb->range);
2623 qemu_aio_unref(iocb);
2624
2625 return status;
2626 }
2627
2628 req->aiocb = &iocb->common;
2629 nvme_dsm_cb(iocb, 0);
2630
2631 return NVME_NO_COMPLETE;
2632 }
2633
2634 return status;
2635 }
2636
2637 static uint16_t nvme_verify(NvmeCtrl *n, NvmeRequest *req)
2638 {
2639 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2640 NvmeNamespace *ns = req->ns;
2641 BlockBackend *blk = ns->blkconf.blk;
2642 uint64_t slba = le64_to_cpu(rw->slba);
2643 uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
2644 size_t len = nvme_l2b(ns, nlb);
2645 int64_t offset = nvme_l2b(ns, slba);
2646 uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
2647 uint32_t reftag = le32_to_cpu(rw->reftag);
2648 NvmeBounceContext *ctx = NULL;
2649 uint16_t status;
2650
2651 trace_pci_nvme_verify(nvme_cid(req), nvme_nsid(ns), slba, nlb);
2652
2653 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
2654 status = nvme_check_prinfo(ns, prinfo, slba, reftag);
2655 if (status) {
2656 return status;
2657 }
2658
2659 if (prinfo & NVME_PRINFO_PRACT) {
2660 return NVME_INVALID_PROT_INFO | NVME_DNR;
2661 }
2662 }
2663
2664 if (len > n->page_size << n->params.vsl) {
2665 return NVME_INVALID_FIELD | NVME_DNR;
2666 }
2667
2668 status = nvme_check_bounds(ns, slba, nlb);
2669 if (status) {
2670 return status;
2671 }
2672
2673 if (NVME_ERR_REC_DULBE(ns->features.err_rec)) {
2674 status = nvme_check_dulbe(ns, slba, nlb);
2675 if (status) {
2676 return status;
2677 }
2678 }
2679
2680 ctx = g_new0(NvmeBounceContext, 1);
2681 ctx->req = req;
2682
2683 ctx->data.bounce = g_malloc(len);
2684
2685 qemu_iovec_init(&ctx->data.iov, 1);
2686 qemu_iovec_add(&ctx->data.iov, ctx->data.bounce, len);
2687
2688 block_acct_start(blk_get_stats(blk), &req->acct, ctx->data.iov.size,
2689 BLOCK_ACCT_READ);
2690
2691 req->aiocb = blk_aio_preadv(ns->blkconf.blk, offset, &ctx->data.iov, 0,
2692 nvme_verify_mdata_in_cb, ctx);
2693 return NVME_NO_COMPLETE;
2694 }
2695
2696 typedef struct NvmeCopyAIOCB {
2697 BlockAIOCB common;
2698 BlockAIOCB *aiocb;
2699 NvmeRequest *req;
2700 int ret;
2701
2702 void *ranges;
2703 unsigned int format;
2704 int nr;
2705 int idx;
2706
2707 uint8_t *bounce;
2708 QEMUIOVector iov;
2709 struct {
2710 BlockAcctCookie read;
2711 BlockAcctCookie write;
2712 } acct;
2713
2714 uint64_t reftag;
2715 uint64_t slba;
2716
2717 NvmeZone *zone;
2718 } NvmeCopyAIOCB;
2719
2720 static void nvme_copy_cancel(BlockAIOCB *aiocb)
2721 {
2722 NvmeCopyAIOCB *iocb = container_of(aiocb, NvmeCopyAIOCB, common);
2723
2724 iocb->ret = -ECANCELED;
2725
2726 if (iocb->aiocb) {
2727 blk_aio_cancel_async(iocb->aiocb);
2728 iocb->aiocb = NULL;
2729 }
2730 }
2731
2732 static const AIOCBInfo nvme_copy_aiocb_info = {
2733 .aiocb_size = sizeof(NvmeCopyAIOCB),
2734 .cancel_async = nvme_copy_cancel,
2735 };
2736
2737 static void nvme_copy_done(NvmeCopyAIOCB *iocb)
2738 {
2739 NvmeRequest *req = iocb->req;
2740 NvmeNamespace *ns = req->ns;
2741 BlockAcctStats *stats = blk_get_stats(ns->blkconf.blk);
2742
2743 if (iocb->idx != iocb->nr) {
2744 req->cqe.result = cpu_to_le32(iocb->idx);
2745 }
2746
2747 qemu_iovec_destroy(&iocb->iov);
2748 g_free(iocb->bounce);
2749
2750 if (iocb->ret < 0) {
2751 block_acct_failed(stats, &iocb->acct.read);
2752 block_acct_failed(stats, &iocb->acct.write);
2753 } else {
2754 block_acct_done(stats, &iocb->acct.read);
2755 block_acct_done(stats, &iocb->acct.write);
2756 }
2757
2758 iocb->common.cb(iocb->common.opaque, iocb->ret);
2759 qemu_aio_unref(iocb);
2760 }
2761
2762 static void nvme_do_copy(NvmeCopyAIOCB *iocb);
2763
2764 static void nvme_copy_source_range_parse_format0(void *ranges, int idx,
2765 uint64_t *slba, uint32_t *nlb,
2766 uint16_t *apptag,
2767 uint16_t *appmask,
2768 uint64_t *reftag)
2769 {
2770 NvmeCopySourceRangeFormat0 *_ranges = ranges;
2771
2772 if (slba) {
2773 *slba = le64_to_cpu(_ranges[idx].slba);
2774 }
2775
2776 if (nlb) {
2777 *nlb = le16_to_cpu(_ranges[idx].nlb) + 1;
2778 }
2779
2780 if (apptag) {
2781 *apptag = le16_to_cpu(_ranges[idx].apptag);
2782 }
2783
2784 if (appmask) {
2785 *appmask = le16_to_cpu(_ranges[idx].appmask);
2786 }
2787
2788 if (reftag) {
2789 *reftag = le32_to_cpu(_ranges[idx].reftag);
2790 }
2791 }
2792
2793 static void nvme_copy_source_range_parse_format1(void *ranges, int idx,
2794 uint64_t *slba, uint32_t *nlb,
2795 uint16_t *apptag,
2796 uint16_t *appmask,
2797 uint64_t *reftag)
2798 {
2799 NvmeCopySourceRangeFormat1 *_ranges = ranges;
2800
2801 if (slba) {
2802 *slba = le64_to_cpu(_ranges[idx].slba);
2803 }
2804
2805 if (nlb) {
2806 *nlb = le16_to_cpu(_ranges[idx].nlb) + 1;
2807 }
2808
2809 if (apptag) {
2810 *apptag = le16_to_cpu(_ranges[idx].apptag);
2811 }
2812
2813 if (appmask) {
2814 *appmask = le16_to_cpu(_ranges[idx].appmask);
2815 }
2816
2817 if (reftag) {
2818 *reftag = 0;
2819
2820 *reftag |= (uint64_t)_ranges[idx].sr[4] << 40;
2821 *reftag |= (uint64_t)_ranges[idx].sr[5] << 32;
2822 *reftag |= (uint64_t)_ranges[idx].sr[6] << 24;
2823 *reftag |= (uint64_t)_ranges[idx].sr[7] << 16;
2824 *reftag |= (uint64_t)_ranges[idx].sr[8] << 8;
2825 *reftag |= (uint64_t)_ranges[idx].sr[9];
2826 }
2827 }
2828
2829 static void nvme_copy_source_range_parse(void *ranges, int idx, uint8_t format,
2830 uint64_t *slba, uint32_t *nlb,
2831 uint16_t *apptag, uint16_t *appmask,
2832 uint64_t *reftag)
2833 {
2834 switch (format) {
2835 case NVME_COPY_FORMAT_0:
2836 nvme_copy_source_range_parse_format0(ranges, idx, slba, nlb, apptag,
2837 appmask, reftag);
2838 break;
2839
2840 case NVME_COPY_FORMAT_1:
2841 nvme_copy_source_range_parse_format1(ranges, idx, slba, nlb, apptag,
2842 appmask, reftag);
2843 break;
2844
2845 default:
2846 abort();
2847 }
2848 }
2849
2850 static void nvme_copy_out_completed_cb(void *opaque, int ret)
2851 {
2852 NvmeCopyAIOCB *iocb = opaque;
2853 NvmeRequest *req = iocb->req;
2854 NvmeNamespace *ns = req->ns;
2855 uint32_t nlb;
2856
2857 nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, NULL,
2858 &nlb, NULL, NULL, NULL);
2859
2860 if (ret < 0) {
2861 iocb->ret = ret;
2862 goto out;
2863 } else if (iocb->ret < 0) {
2864 goto out;
2865 }
2866
2867 if (ns->params.zoned) {
2868 nvme_advance_zone_wp(ns, iocb->zone, nlb);
2869 }
2870
2871 iocb->idx++;
2872 iocb->slba += nlb;
2873 out:
2874 nvme_do_copy(iocb);
2875 }
2876
2877 static void nvme_copy_out_cb(void *opaque, int ret)
2878 {
2879 NvmeCopyAIOCB *iocb = opaque;
2880 NvmeRequest *req = iocb->req;
2881 NvmeNamespace *ns = req->ns;
2882 uint32_t nlb;
2883 size_t mlen;
2884 uint8_t *mbounce;
2885
2886 if (ret < 0 || iocb->ret < 0 || !ns->lbaf.ms) {
2887 goto out;
2888 }
2889
2890 nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, NULL,
2891 &nlb, NULL, NULL, NULL);
2892
2893 mlen = nvme_m2b(ns, nlb);
2894 mbounce = iocb->bounce + nvme_l2b(ns, nlb);
2895
2896 qemu_iovec_reset(&iocb->iov);
2897 qemu_iovec_add(&iocb->iov, mbounce, mlen);
2898
2899 iocb->aiocb = blk_aio_pwritev(ns->blkconf.blk, nvme_moff(ns, iocb->slba),
2900 &iocb->iov, 0, nvme_copy_out_completed_cb,
2901 iocb);
2902
2903 return;
2904
2905 out:
2906 nvme_copy_out_completed_cb(iocb, ret);
2907 }
2908
2909 static void nvme_copy_in_completed_cb(void *opaque, int ret)
2910 {
2911 NvmeCopyAIOCB *iocb = opaque;
2912 NvmeRequest *req = iocb->req;
2913 NvmeNamespace *ns = req->ns;
2914 uint32_t nlb;
2915 uint64_t slba;
2916 uint16_t apptag, appmask;
2917 uint64_t reftag;
2918 size_t len;
2919 uint16_t status;
2920
2921 if (ret < 0) {
2922 iocb->ret = ret;
2923 goto out;
2924 } else if (iocb->ret < 0) {
2925 goto out;
2926 }
2927
2928 nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, &slba,
2929 &nlb, &apptag, &appmask, &reftag);
2930 len = nvme_l2b(ns, nlb);
2931
2932 trace_pci_nvme_copy_out(iocb->slba, nlb);
2933
2934 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
2935 NvmeCopyCmd *copy = (NvmeCopyCmd *)&req->cmd;
2936
2937 uint16_t prinfor = ((copy->control[0] >> 4) & 0xf);
2938 uint16_t prinfow = ((copy->control[2] >> 2) & 0xf);
2939
2940 size_t mlen = nvme_m2b(ns, nlb);
2941 uint8_t *mbounce = iocb->bounce + nvme_l2b(ns, nlb);
2942
2943 status = nvme_dif_mangle_mdata(ns, mbounce, mlen, slba);
2944 if (status) {
2945 goto invalid;
2946 }
2947 status = nvme_dif_check(ns, iocb->bounce, len, mbounce, mlen, prinfor,
2948 slba, apptag, appmask, &reftag);
2949 if (status) {
2950 goto invalid;
2951 }
2952
2953 apptag = le16_to_cpu(copy->apptag);
2954 appmask = le16_to_cpu(copy->appmask);
2955
2956 if (prinfow & NVME_PRINFO_PRACT) {
2957 status = nvme_check_prinfo(ns, prinfow, iocb->slba, iocb->reftag);
2958 if (status) {
2959 goto invalid;
2960 }
2961
2962 nvme_dif_pract_generate_dif(ns, iocb->bounce, len, mbounce, mlen,
2963 apptag, &iocb->reftag);
2964 } else {
2965 status = nvme_dif_check(ns, iocb->bounce, len, mbounce, mlen,
2966 prinfow, iocb->slba, apptag, appmask,
2967 &iocb->reftag);
2968 if (status) {
2969 goto invalid;
2970 }
2971 }
2972 }
2973
2974 status = nvme_check_bounds(ns, iocb->slba, nlb);
2975 if (status) {
2976 goto invalid;
2977 }
2978
2979 if (ns->params.zoned) {
2980 status = nvme_check_zone_write(ns, iocb->zone, iocb->slba, nlb);
2981 if (status) {
2982 goto invalid;
2983 }
2984
2985 if (!(iocb->zone->d.za & NVME_ZA_ZRWA_VALID)) {
2986 iocb->zone->w_ptr += nlb;
2987 }
2988 }
2989
2990 qemu_iovec_reset(&iocb->iov);
2991 qemu_iovec_add(&iocb->iov, iocb->bounce, len);
2992
2993 iocb->aiocb = blk_aio_pwritev(ns->blkconf.blk, nvme_l2b(ns, iocb->slba),
2994 &iocb->iov, 0, nvme_copy_out_cb, iocb);
2995
2996 return;
2997
2998 invalid:
2999 req->status = status;
3000 iocb->ret = -1;
3001 out:
3002 nvme_do_copy(iocb);
3003 }
3004
3005 static void nvme_copy_in_cb(void *opaque, int ret)
3006 {
3007 NvmeCopyAIOCB *iocb = opaque;
3008 NvmeRequest *req = iocb->req;
3009 NvmeNamespace *ns = req->ns;
3010 uint64_t slba;
3011 uint32_t nlb;
3012
3013 if (ret < 0 || iocb->ret < 0 || !ns->lbaf.ms) {
3014 goto out;
3015 }
3016
3017 nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, &slba,
3018 &nlb, NULL, NULL, NULL);
3019
3020 qemu_iovec_reset(&iocb->iov);
3021 qemu_iovec_add(&iocb->iov, iocb->bounce + nvme_l2b(ns, nlb),
3022 nvme_m2b(ns, nlb));
3023
3024 iocb->aiocb = blk_aio_preadv(ns->blkconf.blk, nvme_moff(ns, slba),
3025 &iocb->iov, 0, nvme_copy_in_completed_cb,
3026 iocb);
3027 return;
3028
3029 out:
3030 nvme_copy_in_completed_cb(iocb, ret);
3031 }
3032
3033 static void nvme_do_copy(NvmeCopyAIOCB *iocb)
3034 {
3035 NvmeRequest *req = iocb->req;
3036 NvmeNamespace *ns = req->ns;
3037 uint64_t slba;
3038 uint32_t nlb;
3039 size_t len;
3040 uint16_t status;
3041
3042 if (iocb->ret < 0) {
3043 goto done;
3044 }
3045
3046 if (iocb->idx == iocb->nr) {
3047 goto done;
3048 }
3049
3050 nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, &slba,
3051 &nlb, NULL, NULL, NULL);
3052 len = nvme_l2b(ns, nlb);
3053
3054 trace_pci_nvme_copy_source_range(slba, nlb);
3055
3056 if (nlb > le16_to_cpu(ns->id_ns.mssrl)) {
3057 status = NVME_CMD_SIZE_LIMIT | NVME_DNR;
3058 goto invalid;
3059 }
3060
3061 status = nvme_check_bounds(ns, slba, nlb);
3062 if (status) {
3063 goto invalid;
3064 }
3065
3066 if (NVME_ERR_REC_DULBE(ns->features.err_rec)) {
3067 status = nvme_check_dulbe(ns, slba, nlb);
3068 if (status) {
3069 goto invalid;
3070 }
3071 }
3072
3073 if (ns->params.zoned) {
3074 status = nvme_check_zone_read(ns, slba, nlb);
3075 if (status) {
3076 goto invalid;
3077 }
3078 }
3079
3080 qemu_iovec_reset(&iocb->iov);
3081 qemu_iovec_add(&iocb->iov, iocb->bounce, len);
3082
3083 iocb->aiocb = blk_aio_preadv(ns->blkconf.blk, nvme_l2b(ns, slba),
3084 &iocb->iov, 0, nvme_copy_in_cb, iocb);
3085 return;
3086
3087 invalid:
3088 req->status = status;
3089 iocb->ret = -1;
3090 done:
3091 nvme_copy_done(iocb);
3092 }
3093
3094 static uint16_t nvme_copy(NvmeCtrl *n, NvmeRequest *req)
3095 {
3096 NvmeNamespace *ns = req->ns;
3097 NvmeCopyCmd *copy = (NvmeCopyCmd *)&req->cmd;
3098 NvmeCopyAIOCB *iocb = blk_aio_get(&nvme_copy_aiocb_info, ns->blkconf.blk,
3099 nvme_misc_cb, req);
3100 uint16_t nr = copy->nr + 1;
3101 uint8_t format = copy->control[0] & 0xf;
3102 uint16_t prinfor = ((copy->control[0] >> 4) & 0xf);
3103 uint16_t prinfow = ((copy->control[2] >> 2) & 0xf);
3104 size_t len = sizeof(NvmeCopySourceRangeFormat0);
3105
3106 uint16_t status;
3107
3108 trace_pci_nvme_copy(nvme_cid(req), nvme_nsid(ns), nr, format);
3109
3110 iocb->ranges = NULL;
3111 iocb->zone = NULL;
3112
3113 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) &&
3114 ((prinfor & NVME_PRINFO_PRACT) != (prinfow & NVME_PRINFO_PRACT))) {
3115 status = NVME_INVALID_FIELD | NVME_DNR;
3116 goto invalid;
3117 }
3118
3119 if (!(n->id_ctrl.ocfs & (1 << format))) {
3120 trace_pci_nvme_err_copy_invalid_format(format);
3121 status = NVME_INVALID_FIELD | NVME_DNR;
3122 goto invalid;
3123 }
3124
3125 if (nr > ns->id_ns.msrc + 1) {
3126 status = NVME_CMD_SIZE_LIMIT | NVME_DNR;
3127 goto invalid;
3128 }
3129
3130 if ((ns->pif == 0x0 && format != 0x0) ||
3131 (ns->pif != 0x0 && format != 0x1)) {
3132 status = NVME_INVALID_FORMAT | NVME_DNR;
3133 goto invalid;
3134 }
3135
3136 if (ns->pif) {
3137 len = sizeof(NvmeCopySourceRangeFormat1);
3138 }
3139
3140 iocb->format = format;
3141 iocb->ranges = g_malloc_n(nr, len);
3142 status = nvme_h2c(n, (uint8_t *)iocb->ranges, len * nr, req);
3143 if (status) {
3144 goto invalid;
3145 }
3146
3147 iocb->slba = le64_to_cpu(copy->sdlba);
3148
3149 if (ns->params.zoned) {
3150 iocb->zone = nvme_get_zone_by_slba(ns, iocb->slba);
3151 if (!iocb->zone) {
3152 status = NVME_LBA_RANGE | NVME_DNR;
3153 goto invalid;
3154 }
3155
3156 status = nvme_zrm_auto(n, ns, iocb->zone);
3157 if (status) {
3158 goto invalid;
3159 }
3160 }
3161
3162 iocb->req = req;
3163 iocb->ret = 0;
3164 iocb->nr = nr;
3165 iocb->idx = 0;
3166 iocb->reftag = le32_to_cpu(copy->reftag);
3167 iocb->reftag |= (uint64_t)le32_to_cpu(copy->cdw3) << 32;
3168 iocb->bounce = g_malloc_n(le16_to_cpu(ns->id_ns.mssrl),
3169 ns->lbasz + ns->lbaf.ms);
3170
3171 qemu_iovec_init(&iocb->iov, 1);
3172
3173 block_acct_start(blk_get_stats(ns->blkconf.blk), &iocb->acct.read, 0,
3174 BLOCK_ACCT_READ);
3175 block_acct_start(blk_get_stats(ns->blkconf.blk), &iocb->acct.write, 0,
3176 BLOCK_ACCT_WRITE);
3177
3178 req->aiocb = &iocb->common;
3179 nvme_do_copy(iocb);
3180
3181 return NVME_NO_COMPLETE;
3182
3183 invalid:
3184 g_free(iocb->ranges);
3185 qemu_aio_unref(iocb);
3186 return status;
3187 }
3188
3189 static uint16_t nvme_compare(NvmeCtrl *n, NvmeRequest *req)
3190 {
3191 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
3192 NvmeNamespace *ns = req->ns;
3193 BlockBackend *blk = ns->blkconf.blk;
3194 uint64_t slba = le64_to_cpu(rw->slba);
3195 uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
3196 uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
3197 size_t data_len = nvme_l2b(ns, nlb);
3198 size_t len = data_len;
3199 int64_t offset = nvme_l2b(ns, slba);
3200 struct nvme_compare_ctx *ctx = NULL;
3201 uint16_t status;
3202
3203 trace_pci_nvme_compare(nvme_cid(req), nvme_nsid(ns), slba, nlb);
3204
3205 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) && (prinfo & NVME_PRINFO_PRACT)) {
3206 return NVME_INVALID_PROT_INFO | NVME_DNR;
3207 }
3208
3209 if (nvme_ns_ext(ns)) {
3210 len += nvme_m2b(ns, nlb);
3211 }
3212
3213 status = nvme_check_mdts(n, len);
3214 if (status) {
3215 return status;
3216 }
3217
3218 status = nvme_check_bounds(ns, slba, nlb);
3219 if (status) {
3220 return status;
3221 }
3222
3223 if (NVME_ERR_REC_DULBE(ns->features.err_rec)) {
3224 status = nvme_check_dulbe(ns, slba, nlb);
3225 if (status) {
3226 return status;
3227 }
3228 }
3229
3230 status = nvme_map_dptr(n, &req->sg, len, &req->cmd);
3231 if (status) {
3232 return status;
3233 }
3234
3235 ctx = g_new(struct nvme_compare_ctx, 1);
3236 ctx->data.bounce = g_malloc(data_len);
3237
3238 req->opaque = ctx;
3239
3240 qemu_iovec_init(&ctx->data.iov, 1);
3241 qemu_iovec_add(&ctx->data.iov, ctx->data.bounce, data_len);
3242
3243 block_acct_start(blk_get_stats(blk), &req->acct, data_len,
3244 BLOCK_ACCT_READ);
3245 req->aiocb = blk_aio_preadv(blk, offset, &ctx->data.iov, 0,
3246 nvme_compare_data_cb, req);
3247
3248 return NVME_NO_COMPLETE;
3249 }
3250
3251 typedef struct NvmeFlushAIOCB {
3252 BlockAIOCB common;
3253 BlockAIOCB *aiocb;
3254 NvmeRequest *req;
3255 int ret;
3256
3257 NvmeNamespace *ns;
3258 uint32_t nsid;
3259 bool broadcast;
3260 } NvmeFlushAIOCB;
3261
3262 static void nvme_flush_cancel(BlockAIOCB *acb)
3263 {
3264 NvmeFlushAIOCB *iocb = container_of(acb, NvmeFlushAIOCB, common);
3265
3266 iocb->ret = -ECANCELED;
3267
3268 if (iocb->aiocb) {
3269 blk_aio_cancel_async(iocb->aiocb);
3270 iocb->aiocb = NULL;
3271 }
3272 }
3273
3274 static const AIOCBInfo nvme_flush_aiocb_info = {
3275 .aiocb_size = sizeof(NvmeFlushAIOCB),
3276 .cancel_async = nvme_flush_cancel,
3277 .get_aio_context = nvme_get_aio_context,
3278 };
3279
3280 static void nvme_do_flush(NvmeFlushAIOCB *iocb);
3281
3282 static void nvme_flush_ns_cb(void *opaque, int ret)
3283 {
3284 NvmeFlushAIOCB *iocb = opaque;
3285 NvmeNamespace *ns = iocb->ns;
3286
3287 if (ret < 0) {
3288 iocb->ret = ret;
3289 goto out;
3290 } else if (iocb->ret < 0) {
3291 goto out;
3292 }
3293
3294 if (ns) {
3295 trace_pci_nvme_flush_ns(iocb->nsid);
3296
3297 iocb->ns = NULL;
3298 iocb->aiocb = blk_aio_flush(ns->blkconf.blk, nvme_flush_ns_cb, iocb);
3299 return;
3300 }
3301
3302 out:
3303 nvme_do_flush(iocb);
3304 }
3305
3306 static void nvme_do_flush(NvmeFlushAIOCB *iocb)
3307 {
3308 NvmeRequest *req = iocb->req;
3309 NvmeCtrl *n = nvme_ctrl(req);
3310 int i;
3311
3312 if (iocb->ret < 0) {
3313 goto done;
3314 }
3315
3316 if (iocb->broadcast) {
3317 for (i = iocb->nsid + 1; i <= NVME_MAX_NAMESPACES; i++) {
3318 iocb->ns = nvme_ns(n, i);
3319 if (iocb->ns) {
3320 iocb->nsid = i;
3321 break;
3322 }
3323 }
3324 }
3325
3326 if (!iocb->ns) {
3327 goto done;
3328 }
3329
3330 nvme_flush_ns_cb(iocb, 0);
3331 return;
3332
3333 done:
3334 iocb->common.cb(iocb->common.opaque, iocb->ret);
3335 qemu_aio_unref(iocb);
3336 }
3337
3338 static uint16_t nvme_flush(NvmeCtrl *n, NvmeRequest *req)
3339 {
3340 NvmeFlushAIOCB *iocb;
3341 uint32_t nsid = le32_to_cpu(req->cmd.nsid);
3342 uint16_t status;
3343
3344 iocb = qemu_aio_get(&nvme_flush_aiocb_info, NULL, nvme_misc_cb, req);
3345
3346 iocb->req = req;
3347 iocb->ret = 0;
3348 iocb->ns = NULL;
3349 iocb->nsid = 0;
3350 iocb->broadcast = (nsid == NVME_NSID_BROADCAST);
3351
3352 if (!iocb->broadcast) {
3353 if (!nvme_nsid_valid(n, nsid)) {
3354 status = NVME_INVALID_NSID | NVME_DNR;
3355 goto out;
3356 }
3357
3358 iocb->ns = nvme_ns(n, nsid);
3359 if (!iocb->ns) {
3360 status = NVME_INVALID_FIELD | NVME_DNR;
3361 goto out;
3362 }
3363
3364 iocb->nsid = nsid;
3365 }
3366
3367 req->aiocb = &iocb->common;
3368 nvme_do_flush(iocb);
3369
3370 return NVME_NO_COMPLETE;
3371
3372 out:
3373 qemu_aio_unref(iocb);
3374
3375 return status;
3376 }
3377
3378 static uint16_t nvme_read(NvmeCtrl *n, NvmeRequest *req)
3379 {
3380 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
3381 NvmeNamespace *ns = req->ns;
3382 uint64_t slba = le64_to_cpu(rw->slba);
3383 uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb) + 1;
3384 uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
3385 uint64_t data_size = nvme_l2b(ns, nlb);
3386 uint64_t mapped_size = data_size;
3387 uint64_t data_offset;
3388 BlockBackend *blk = ns->blkconf.blk;
3389 uint16_t status;
3390
3391 if (nvme_ns_ext(ns)) {
3392 mapped_size += nvme_m2b(ns, nlb);
3393
3394 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3395 bool pract = prinfo & NVME_PRINFO_PRACT;
3396
3397 if (pract && ns->lbaf.ms == nvme_pi_tuple_size(ns)) {
3398 mapped_size = data_size;
3399 }
3400 }
3401 }
3402
3403 trace_pci_nvme_read(nvme_cid(req), nvme_nsid(ns), nlb, mapped_size, slba);
3404
3405 status = nvme_check_mdts(n, mapped_size);
3406 if (status) {
3407 goto invalid;
3408 }
3409
3410 status = nvme_check_bounds(ns, slba, nlb);
3411 if (status) {
3412 goto invalid;
3413 }
3414
3415 if (ns->params.zoned) {
3416 status = nvme_check_zone_read(ns, slba, nlb);
3417 if (status) {
3418 trace_pci_nvme_err_zone_read_not_ok(slba, nlb, status);
3419 goto invalid;
3420 }
3421 }
3422
3423 if (NVME_ERR_REC_DULBE(ns->features.err_rec)) {
3424 status = nvme_check_dulbe(ns, slba, nlb);
3425 if (status) {
3426 goto invalid;
3427 }
3428 }
3429
3430 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3431 return nvme_dif_rw(n, req);
3432 }
3433
3434 status = nvme_map_data(n, nlb, req);
3435 if (status) {
3436 goto invalid;
3437 }
3438
3439 data_offset = nvme_l2b(ns, slba);
3440
3441 block_acct_start(blk_get_stats(blk), &req->acct, data_size,
3442 BLOCK_ACCT_READ);
3443 nvme_blk_read(blk, data_offset, BDRV_SECTOR_SIZE, nvme_rw_cb, req);
3444 return NVME_NO_COMPLETE;
3445
3446 invalid:
3447 block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_READ);
3448 return status | NVME_DNR;
3449 }
3450
3451 static void nvme_do_write_fdp(NvmeCtrl *n, NvmeRequest *req, uint64_t slba,
3452 uint32_t nlb)
3453 {
3454 NvmeNamespace *ns = req->ns;
3455 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
3456 uint64_t data_size = nvme_l2b(ns, nlb);
3457 uint32_t dw12 = le32_to_cpu(req->cmd.cdw12);
3458 uint8_t dtype = (dw12 >> 20) & 0xf;
3459 uint16_t pid = le16_to_cpu(rw->dspec);
3460 uint16_t ph, rg, ruhid;
3461 NvmeReclaimUnit *ru;
3462
3463 if (dtype != NVME_DIRECTIVE_DATA_PLACEMENT ||
3464 !nvme_parse_pid(ns, pid, &ph, &rg)) {
3465 ph = 0;
3466 rg = 0;
3467 }
3468
3469 ruhid = ns->fdp.phs[ph];
3470 ru = &ns->endgrp->fdp.ruhs[ruhid].rus[rg];
3471
3472 nvme_fdp_stat_inc(&ns->endgrp->fdp.hbmw, data_size);
3473 nvme_fdp_stat_inc(&ns->endgrp->fdp.mbmw, data_size);
3474
3475 while (nlb) {
3476 if (nlb < ru->ruamw) {
3477 ru->ruamw -= nlb;
3478 break;
3479 }
3480
3481 nlb -= ru->ruamw;
3482 nvme_update_ruh(n, ns, pid);
3483 }
3484 }
3485
3486 static uint16_t nvme_do_write(NvmeCtrl *n, NvmeRequest *req, bool append,
3487 bool wrz)
3488 {
3489 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
3490 NvmeNamespace *ns = req->ns;
3491 uint64_t slba = le64_to_cpu(rw->slba);
3492 uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb) + 1;
3493 uint16_t ctrl = le16_to_cpu(rw->control);
3494 uint8_t prinfo = NVME_RW_PRINFO(ctrl);
3495 uint64_t data_size = nvme_l2b(ns, nlb);
3496 uint64_t mapped_size = data_size;
3497 uint64_t data_offset;
3498 NvmeZone *zone;
3499 NvmeZonedResult *res = (NvmeZonedResult *)&req->cqe;
3500 BlockBackend *blk = ns->blkconf.blk;
3501 uint16_t status;
3502
3503 if (nvme_ns_ext(ns)) {
3504 mapped_size += nvme_m2b(ns, nlb);
3505
3506 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3507 bool pract = prinfo & NVME_PRINFO_PRACT;
3508
3509 if (pract && ns->lbaf.ms == nvme_pi_tuple_size(ns)) {
3510 mapped_size -= nvme_m2b(ns, nlb);
3511 }
3512 }
3513 }
3514
3515 trace_pci_nvme_write(nvme_cid(req), nvme_io_opc_str(rw->opcode),
3516 nvme_nsid(ns), nlb, mapped_size, slba);
3517
3518 if (!wrz) {
3519 status = nvme_check_mdts(n, mapped_size);
3520 if (status) {
3521 goto invalid;
3522 }
3523 }
3524
3525 status = nvme_check_bounds(ns, slba, nlb);
3526 if (status) {
3527 goto invalid;
3528 }
3529
3530 if (ns->params.zoned) {
3531 zone = nvme_get_zone_by_slba(ns, slba);
3532 assert(zone);
3533
3534 if (append) {
3535 bool piremap = !!(ctrl & NVME_RW_PIREMAP);
3536
3537 if (unlikely(zone->d.za & NVME_ZA_ZRWA_VALID)) {
3538 return NVME_INVALID_ZONE_OP | NVME_DNR;
3539 }
3540
3541 if (unlikely(slba != zone->d.zslba)) {
3542 trace_pci_nvme_err_append_not_at_start(slba, zone->d.zslba);
3543 status = NVME_INVALID_FIELD;
3544 goto invalid;
3545 }
3546
3547 if (n->params.zasl &&
3548 data_size > (uint64_t)n->page_size << n->params.zasl) {
3549 trace_pci_nvme_err_zasl(data_size);
3550 return NVME_INVALID_FIELD | NVME_DNR;
3551 }
3552
3553 slba = zone->w_ptr;
3554 rw->slba = cpu_to_le64(slba);
3555 res->slba = cpu_to_le64(slba);
3556
3557 switch (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3558 case NVME_ID_NS_DPS_TYPE_1:
3559 if (!piremap) {
3560 return NVME_INVALID_PROT_INFO | NVME_DNR;
3561 }
3562
3563 /* fallthrough */
3564
3565 case NVME_ID_NS_DPS_TYPE_2:
3566 if (piremap) {
3567 uint32_t reftag = le32_to_cpu(rw->reftag);
3568 rw->reftag = cpu_to_le32(reftag + (slba - zone->d.zslba));
3569 }
3570
3571 break;
3572
3573 case NVME_ID_NS_DPS_TYPE_3:
3574 if (piremap) {
3575 return NVME_INVALID_PROT_INFO | NVME_DNR;
3576 }
3577
3578 break;
3579 }
3580 }
3581
3582 status = nvme_check_zone_write(ns, zone, slba, nlb);
3583 if (status) {
3584 goto invalid;
3585 }
3586
3587 status = nvme_zrm_auto(n, ns, zone);
3588 if (status) {
3589 goto invalid;
3590 }
3591
3592 if (!(zone->d.za & NVME_ZA_ZRWA_VALID)) {
3593 zone->w_ptr += nlb;
3594 }
3595 } else if (ns->endgrp && ns->endgrp->fdp.enabled) {
3596 nvme_do_write_fdp(n, req, slba, nlb);
3597 }
3598
3599 data_offset = nvme_l2b(ns, slba);
3600
3601 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3602 return nvme_dif_rw(n, req);
3603 }
3604
3605 if (!wrz) {
3606 status = nvme_map_data(n, nlb, req);
3607 if (status) {
3608 goto invalid;
3609 }
3610
3611 block_acct_start(blk_get_stats(blk), &req->acct, data_size,
3612 BLOCK_ACCT_WRITE);
3613 nvme_blk_write(blk, data_offset, BDRV_SECTOR_SIZE, nvme_rw_cb, req);
3614 } else {
3615 req->aiocb = blk_aio_pwrite_zeroes(blk, data_offset, data_size,
3616 BDRV_REQ_MAY_UNMAP, nvme_rw_cb,
3617 req);
3618 }
3619
3620 return NVME_NO_COMPLETE;
3621
3622 invalid:
3623 block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_WRITE);
3624 return status | NVME_DNR;
3625 }
3626
3627 static inline uint16_t nvme_write(NvmeCtrl *n, NvmeRequest *req)
3628 {
3629 return nvme_do_write(n, req, false, false);
3630 }
3631
3632 static inline uint16_t nvme_write_zeroes(NvmeCtrl *n, NvmeRequest *req)
3633 {
3634 return nvme_do_write(n, req, false, true);
3635 }
3636
3637 static inline uint16_t nvme_zone_append(NvmeCtrl *n, NvmeRequest *req)
3638 {
3639 return nvme_do_write(n, req, true, false);
3640 }
3641
3642 static uint16_t nvme_get_mgmt_zone_slba_idx(NvmeNamespace *ns, NvmeCmd *c,
3643 uint64_t *slba, uint32_t *zone_idx)
3644 {
3645 uint32_t dw10 = le32_to_cpu(c->cdw10);
3646 uint32_t dw11 = le32_to_cpu(c->cdw11);
3647
3648 if (!ns->params.zoned) {
3649 trace_pci_nvme_err_invalid_opc(c->opcode);
3650 return NVME_INVALID_OPCODE | NVME_DNR;
3651 }
3652
3653 *slba = ((uint64_t)dw11) << 32 | dw10;
3654 if (unlikely(*slba >= ns->id_ns.nsze)) {
3655 trace_pci_nvme_err_invalid_lba_range(*slba, 0, ns->id_ns.nsze);
3656 *slba = 0;
3657 return NVME_LBA_RANGE | NVME_DNR;
3658 }
3659
3660 *zone_idx = nvme_zone_idx(ns, *slba);
3661 assert(*zone_idx < ns->num_zones);
3662
3663 return NVME_SUCCESS;
3664 }
3665
3666 typedef uint16_t (*op_handler_t)(NvmeNamespace *, NvmeZone *, NvmeZoneState,
3667 NvmeRequest *);
3668
3669 enum NvmeZoneProcessingMask {
3670 NVME_PROC_CURRENT_ZONE = 0,
3671 NVME_PROC_OPENED_ZONES = 1 << 0,
3672 NVME_PROC_CLOSED_ZONES = 1 << 1,
3673 NVME_PROC_READ_ONLY_ZONES = 1 << 2,
3674 NVME_PROC_FULL_ZONES = 1 << 3,
3675 };
3676
3677 static uint16_t nvme_open_zone(NvmeNamespace *ns, NvmeZone *zone,
3678 NvmeZoneState state, NvmeRequest *req)
3679 {
3680 NvmeZoneSendCmd *cmd = (NvmeZoneSendCmd *)&req->cmd;
3681 int flags = 0;
3682
3683 if (cmd->zsflags & NVME_ZSFLAG_ZRWA_ALLOC) {
3684 uint16_t ozcs = le16_to_cpu(ns->id_ns_zoned->ozcs);
3685
3686 if (!(ozcs & NVME_ID_NS_ZONED_OZCS_ZRWASUP)) {
3687 return NVME_INVALID_ZONE_OP | NVME_DNR;
3688 }
3689
3690 if (zone->w_ptr % ns->zns.zrwafg) {
3691 return NVME_NOZRWA | NVME_DNR;
3692 }
3693
3694 flags = NVME_ZRM_ZRWA;
3695 }
3696
3697 return nvme_zrm_open_flags(nvme_ctrl(req), ns, zone, flags);
3698 }
3699
3700 static uint16_t nvme_close_zone(NvmeNamespace *ns, NvmeZone *zone,
3701 NvmeZoneState state, NvmeRequest *req)
3702 {
3703 return nvme_zrm_close(ns, zone);
3704 }
3705
3706 static uint16_t nvme_finish_zone(NvmeNamespace *ns, NvmeZone *zone,
3707 NvmeZoneState state, NvmeRequest *req)
3708 {
3709 return nvme_zrm_finish(ns, zone);
3710 }
3711
3712 static uint16_t nvme_offline_zone(NvmeNamespace *ns, NvmeZone *zone,
3713 NvmeZoneState state, NvmeRequest *req)
3714 {
3715 switch (state) {
3716 case NVME_ZONE_STATE_READ_ONLY:
3717 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_OFFLINE);
3718 /* fall through */
3719 case NVME_ZONE_STATE_OFFLINE:
3720 return NVME_SUCCESS;
3721 default:
3722 return NVME_ZONE_INVAL_TRANSITION;
3723 }
3724 }
3725
3726 static uint16_t nvme_set_zd_ext(NvmeNamespace *ns, NvmeZone *zone)
3727 {
3728 uint16_t status;
3729 uint8_t state = nvme_get_zone_state(zone);
3730
3731 if (state == NVME_ZONE_STATE_EMPTY) {
3732 status = nvme_aor_check(ns, 1, 0);
3733 if (status) {
3734 return status;
3735 }
3736 nvme_aor_inc_active(ns);
3737 zone->d.za |= NVME_ZA_ZD_EXT_VALID;
3738 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_CLOSED);
3739 return NVME_SUCCESS;
3740 }
3741
3742 return NVME_ZONE_INVAL_TRANSITION;
3743 }
3744
3745 static uint16_t nvme_bulk_proc_zone(NvmeNamespace *ns, NvmeZone *zone,
3746 enum NvmeZoneProcessingMask proc_mask,
3747 op_handler_t op_hndlr, NvmeRequest *req)
3748 {
3749 uint16_t status = NVME_SUCCESS;
3750 NvmeZoneState zs = nvme_get_zone_state(zone);
3751 bool proc_zone;
3752
3753 switch (zs) {
3754 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
3755 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
3756 proc_zone = proc_mask & NVME_PROC_OPENED_ZONES;
3757 break;
3758 case NVME_ZONE_STATE_CLOSED:
3759 proc_zone = proc_mask & NVME_PROC_CLOSED_ZONES;
3760 break;
3761 case NVME_ZONE_STATE_READ_ONLY:
3762 proc_zone = proc_mask & NVME_PROC_READ_ONLY_ZONES;
3763 break;
3764 case NVME_ZONE_STATE_FULL:
3765 proc_zone = proc_mask & NVME_PROC_FULL_ZONES;
3766 break;
3767 default:
3768 proc_zone = false;
3769 }
3770
3771 if (proc_zone) {
3772 status = op_hndlr(ns, zone, zs, req);
3773 }
3774
3775 return status;
3776 }
3777
3778 static uint16_t nvme_do_zone_op(NvmeNamespace *ns, NvmeZone *zone,
3779 enum NvmeZoneProcessingMask proc_mask,
3780 op_handler_t op_hndlr, NvmeRequest *req)
3781 {
3782 NvmeZone *next;
3783 uint16_t status = NVME_SUCCESS;
3784 int i;
3785
3786 if (!proc_mask) {
3787 status = op_hndlr(ns, zone, nvme_get_zone_state(zone), req);
3788 } else {
3789 if (proc_mask & NVME_PROC_CLOSED_ZONES) {
3790 QTAILQ_FOREACH_SAFE(zone, &ns->closed_zones, entry, next) {
3791 status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
3792 req);
3793 if (status && status != NVME_NO_COMPLETE) {
3794 goto out;
3795 }
3796 }
3797 }
3798 if (proc_mask & NVME_PROC_OPENED_ZONES) {
3799 QTAILQ_FOREACH_SAFE(zone, &ns->imp_open_zones, entry, next) {
3800 status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
3801 req);
3802 if (status && status != NVME_NO_COMPLETE) {
3803 goto out;
3804 }
3805 }
3806
3807 QTAILQ_FOREACH_SAFE(zone, &ns->exp_open_zones, entry, next) {
3808 status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
3809 req);
3810 if (status && status != NVME_NO_COMPLETE) {
3811 goto out;
3812 }
3813 }
3814 }
3815 if (proc_mask & NVME_PROC_FULL_ZONES) {
3816 QTAILQ_FOREACH_SAFE(zone, &ns->full_zones, entry, next) {
3817 status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
3818 req);
3819 if (status && status != NVME_NO_COMPLETE) {
3820 goto out;
3821 }
3822 }
3823 }
3824
3825 if (proc_mask & NVME_PROC_READ_ONLY_ZONES) {
3826 for (i = 0; i < ns->num_zones; i++, zone++) {
3827 status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
3828 req);
3829 if (status && status != NVME_NO_COMPLETE) {
3830 goto out;
3831 }
3832 }
3833 }
3834 }
3835
3836 out:
3837 return status;
3838 }
3839
3840 typedef struct NvmeZoneResetAIOCB {
3841 BlockAIOCB common;
3842 BlockAIOCB *aiocb;
3843 NvmeRequest *req;
3844 int ret;
3845
3846 bool all;
3847 int idx;
3848 NvmeZone *zone;
3849 } NvmeZoneResetAIOCB;
3850
3851 static void nvme_zone_reset_cancel(BlockAIOCB *aiocb)
3852 {
3853 NvmeZoneResetAIOCB *iocb = container_of(aiocb, NvmeZoneResetAIOCB, common);
3854 NvmeRequest *req = iocb->req;
3855 NvmeNamespace *ns = req->ns;
3856
3857 iocb->idx = ns->num_zones;
3858
3859 iocb->ret = -ECANCELED;
3860
3861 if (iocb->aiocb) {
3862 blk_aio_cancel_async(iocb->aiocb);
3863 iocb->aiocb = NULL;
3864 }
3865 }
3866
3867 static const AIOCBInfo nvme_zone_reset_aiocb_info = {
3868 .aiocb_size = sizeof(NvmeZoneResetAIOCB),
3869 .cancel_async = nvme_zone_reset_cancel,
3870 };
3871
3872 static void nvme_zone_reset_cb(void *opaque, int ret);
3873
3874 static void nvme_zone_reset_epilogue_cb(void *opaque, int ret)
3875 {
3876 NvmeZoneResetAIOCB *iocb = opaque;
3877 NvmeRequest *req = iocb->req;
3878 NvmeNamespace *ns = req->ns;
3879 int64_t moff;
3880 int count;
3881
3882 if (ret < 0 || iocb->ret < 0 || !ns->lbaf.ms) {
3883 goto out;
3884 }
3885
3886 moff = nvme_moff(ns, iocb->zone->d.zslba);
3887 count = nvme_m2b(ns, ns->zone_size);
3888
3889 iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk, moff, count,
3890 BDRV_REQ_MAY_UNMAP,
3891 nvme_zone_reset_cb, iocb);
3892 return;
3893
3894 out:
3895 nvme_zone_reset_cb(iocb, ret);
3896 }
3897
3898 static void nvme_zone_reset_cb(void *opaque, int ret)
3899 {
3900 NvmeZoneResetAIOCB *iocb = opaque;
3901 NvmeRequest *req = iocb->req;
3902 NvmeNamespace *ns = req->ns;
3903
3904 if (iocb->ret < 0) {
3905 goto done;
3906 } else if (ret < 0) {
3907 iocb->ret = ret;
3908 goto done;
3909 }
3910
3911 if (iocb->zone) {
3912 nvme_zrm_reset(ns, iocb->zone);
3913
3914 if (!iocb->all) {
3915 goto done;
3916 }
3917 }
3918
3919 while (iocb->idx < ns->num_zones) {
3920 NvmeZone *zone = &ns->zone_array[iocb->idx++];
3921
3922 switch (nvme_get_zone_state(zone)) {
3923 case NVME_ZONE_STATE_EMPTY:
3924 if (!iocb->all) {
3925 goto done;
3926 }
3927
3928 continue;
3929
3930 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
3931 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
3932 case NVME_ZONE_STATE_CLOSED:
3933 case NVME_ZONE_STATE_FULL:
3934 iocb->zone = zone;
3935 break;
3936
3937 default:
3938 continue;
3939 }
3940
3941 trace_pci_nvme_zns_zone_reset(zone->d.zslba);
3942
3943 iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk,
3944 nvme_l2b(ns, zone->d.zslba),
3945 nvme_l2b(ns, ns->zone_size),
3946 BDRV_REQ_MAY_UNMAP,
3947 nvme_zone_reset_epilogue_cb,
3948 iocb);
3949 return;
3950 }
3951
3952 done:
3953 iocb->aiocb = NULL;
3954
3955 iocb->common.cb(iocb->common.opaque, iocb->ret);
3956 qemu_aio_unref(iocb);
3957 }
3958
3959 static uint16_t nvme_zone_mgmt_send_zrwa_flush(NvmeCtrl *n, NvmeZone *zone,
3960 uint64_t elba, NvmeRequest *req)
3961 {
3962 NvmeNamespace *ns = req->ns;
3963 uint16_t ozcs = le16_to_cpu(ns->id_ns_zoned->ozcs);
3964 uint64_t wp = zone->d.wp;
3965 uint32_t nlb = elba - wp + 1;
3966 uint16_t status;
3967
3968
3969 if (!(ozcs & NVME_ID_NS_ZONED_OZCS_ZRWASUP)) {
3970 return NVME_INVALID_ZONE_OP | NVME_DNR;
3971 }
3972
3973 if (!(zone->d.za & NVME_ZA_ZRWA_VALID)) {
3974 return NVME_INVALID_FIELD | NVME_DNR;
3975 }
3976
3977 if (elba < wp || elba > wp + ns->zns.zrwas) {
3978 return NVME_ZONE_BOUNDARY_ERROR | NVME_DNR;
3979 }
3980
3981 if (nlb % ns->zns.zrwafg) {
3982 return NVME_INVALID_FIELD | NVME_DNR;
3983 }
3984
3985 status = nvme_zrm_auto(n, ns, zone);
3986 if (status) {
3987 return status;
3988 }
3989
3990 zone->w_ptr += nlb;
3991
3992 nvme_advance_zone_wp(ns, zone, nlb);
3993
3994 return NVME_SUCCESS;
3995 }
3996
3997 static uint16_t nvme_zone_mgmt_send(NvmeCtrl *n, NvmeRequest *req)
3998 {
3999 NvmeZoneSendCmd *cmd = (NvmeZoneSendCmd *)&req->cmd;
4000 NvmeNamespace *ns = req->ns;
4001 NvmeZone *zone;
4002 NvmeZoneResetAIOCB *iocb;
4003 uint8_t *zd_ext;
4004 uint64_t slba = 0;
4005 uint32_t zone_idx = 0;
4006 uint16_t status;
4007 uint8_t action = cmd->zsa;
4008 bool all;
4009 enum NvmeZoneProcessingMask proc_mask = NVME_PROC_CURRENT_ZONE;
4010
4011 all = cmd->zsflags & NVME_ZSFLAG_SELECT_ALL;
4012
4013 req->status = NVME_SUCCESS;
4014
4015 if (!all) {
4016 status = nvme_get_mgmt_zone_slba_idx(ns, &req->cmd, &slba, &zone_idx);
4017 if (status) {
4018 return status;
4019 }
4020 }
4021
4022 zone = &ns->zone_array[zone_idx];
4023 if (slba != zone->d.zslba && action != NVME_ZONE_ACTION_ZRWA_FLUSH) {
4024 trace_pci_nvme_err_unaligned_zone_cmd(action, slba, zone->d.zslba);
4025 return NVME_INVALID_FIELD | NVME_DNR;
4026 }
4027
4028 switch (action) {
4029
4030 case NVME_ZONE_ACTION_OPEN:
4031 if (all) {
4032 proc_mask = NVME_PROC_CLOSED_ZONES;
4033 }
4034 trace_pci_nvme_open_zone(slba, zone_idx, all);
4035 status = nvme_do_zone_op(ns, zone, proc_mask, nvme_open_zone, req);
4036 break;
4037
4038 case NVME_ZONE_ACTION_CLOSE:
4039 if (all) {
4040 proc_mask = NVME_PROC_OPENED_ZONES;
4041 }
4042 trace_pci_nvme_close_zone(slba, zone_idx, all);
4043 status = nvme_do_zone_op(ns, zone, proc_mask, nvme_close_zone, req);
4044 break;
4045
4046 case NVME_ZONE_ACTION_FINISH:
4047 if (all) {
4048 proc_mask = NVME_PROC_OPENED_ZONES | NVME_PROC_CLOSED_ZONES;
4049 }
4050 trace_pci_nvme_finish_zone(slba, zone_idx, all);
4051 status = nvme_do_zone_op(ns, zone, proc_mask, nvme_finish_zone, req);
4052 break;
4053
4054 case NVME_ZONE_ACTION_RESET:
4055 trace_pci_nvme_reset_zone(slba, zone_idx, all);
4056
4057 iocb = blk_aio_get(&nvme_zone_reset_aiocb_info, ns->blkconf.blk,
4058 nvme_misc_cb, req);
4059
4060 iocb->req = req;
4061 iocb->ret = 0;
4062 iocb->all = all;
4063 iocb->idx = zone_idx;
4064 iocb->zone = NULL;
4065
4066 req->aiocb = &iocb->common;
4067 nvme_zone_reset_cb(iocb, 0);
4068
4069 return NVME_NO_COMPLETE;
4070
4071 case NVME_ZONE_ACTION_OFFLINE:
4072 if (all) {
4073 proc_mask = NVME_PROC_READ_ONLY_ZONES;
4074 }
4075 trace_pci_nvme_offline_zone(slba, zone_idx, all);
4076 status = nvme_do_zone_op(ns, zone, proc_mask, nvme_offline_zone, req);
4077 break;
4078
4079 case NVME_ZONE_ACTION_SET_ZD_EXT:
4080 trace_pci_nvme_set_descriptor_extension(slba, zone_idx);
4081 if (all || !ns->params.zd_extension_size) {
4082 return NVME_INVALID_FIELD | NVME_DNR;
4083 }
4084 zd_ext = nvme_get_zd_extension(ns, zone_idx);
4085 status = nvme_h2c(n, zd_ext, ns->params.zd_extension_size, req);
4086 if (status) {
4087 trace_pci_nvme_err_zd_extension_map_error(zone_idx);
4088 return status;
4089 }
4090
4091 status = nvme_set_zd_ext(ns, zone);
4092 if (status == NVME_SUCCESS) {
4093 trace_pci_nvme_zd_extension_set(zone_idx);
4094 return status;
4095 }
4096 break;
4097
4098 case NVME_ZONE_ACTION_ZRWA_FLUSH:
4099 if (all) {
4100 return NVME_INVALID_FIELD | NVME_DNR;
4101 }
4102
4103 return nvme_zone_mgmt_send_zrwa_flush(n, zone, slba, req);
4104
4105 default:
4106 trace_pci_nvme_err_invalid_mgmt_action(action);
4107 status = NVME_INVALID_FIELD;
4108 }
4109
4110 if (status == NVME_ZONE_INVAL_TRANSITION) {
4111 trace_pci_nvme_err_invalid_zone_state_transition(action, slba,
4112 zone->d.za);
4113 }
4114 if (status) {
4115 status |= NVME_DNR;
4116 }
4117
4118 return status;
4119 }
4120
4121 static bool nvme_zone_matches_filter(uint32_t zafs, NvmeZone *zl)
4122 {
4123 NvmeZoneState zs = nvme_get_zone_state(zl);
4124
4125 switch (zafs) {
4126 case NVME_ZONE_REPORT_ALL:
4127 return true;
4128 case NVME_ZONE_REPORT_EMPTY:
4129 return zs == NVME_ZONE_STATE_EMPTY;
4130 case NVME_ZONE_REPORT_IMPLICITLY_OPEN:
4131 return zs == NVME_ZONE_STATE_IMPLICITLY_OPEN;
4132 case NVME_ZONE_REPORT_EXPLICITLY_OPEN:
4133 return zs == NVME_ZONE_STATE_EXPLICITLY_OPEN;
4134 case NVME_ZONE_REPORT_CLOSED:
4135 return zs == NVME_ZONE_STATE_CLOSED;
4136 case NVME_ZONE_REPORT_FULL:
4137 return zs == NVME_ZONE_STATE_FULL;
4138 case NVME_ZONE_REPORT_READ_ONLY:
4139 return zs == NVME_ZONE_STATE_READ_ONLY;
4140 case NVME_ZONE_REPORT_OFFLINE:
4141 return zs == NVME_ZONE_STATE_OFFLINE;
4142 default:
4143 return false;
4144 }
4145 }
4146
4147 static uint16_t nvme_zone_mgmt_recv(NvmeCtrl *n, NvmeRequest *req)
4148 {
4149 NvmeCmd *cmd = (NvmeCmd *)&req->cmd;
4150 NvmeNamespace *ns = req->ns;
4151 /* cdw12 is zero-based number of dwords to return. Convert to bytes */
4152 uint32_t data_size = (le32_to_cpu(cmd->cdw12) + 1) << 2;
4153 uint32_t dw13 = le32_to_cpu(cmd->cdw13);
4154 uint32_t zone_idx, zra, zrasf, partial;
4155 uint64_t max_zones, nr_zones = 0;
4156 uint16_t status;
4157 uint64_t slba;
4158 NvmeZoneDescr *z;
4159 NvmeZone *zone;
4160 NvmeZoneReportHeader *header;
4161 void *buf, *buf_p;
4162 size_t zone_entry_sz;
4163 int i;
4164
4165 req->status = NVME_SUCCESS;
4166
4167 status = nvme_get_mgmt_zone_slba_idx(ns, cmd, &slba, &zone_idx);
4168 if (status) {
4169 return status;
4170 }
4171
4172 zra = dw13 & 0xff;
4173 if (zra != NVME_ZONE_REPORT && zra != NVME_ZONE_REPORT_EXTENDED) {
4174 return NVME_INVALID_FIELD | NVME_DNR;
4175 }
4176 if (zra == NVME_ZONE_REPORT_EXTENDED && !ns->params.zd_extension_size) {
4177 return NVME_INVALID_FIELD | NVME_DNR;
4178 }
4179
4180 zrasf = (dw13 >> 8) & 0xff;
4181 if (zrasf > NVME_ZONE_REPORT_OFFLINE) {
4182 return NVME_INVALID_FIELD | NVME_DNR;
4183 }
4184
4185 if (data_size < sizeof(NvmeZoneReportHeader)) {
4186 return NVME_INVALID_FIELD | NVME_DNR;
4187 }
4188
4189 status = nvme_check_mdts(n, data_size);
4190 if (status) {
4191 return status;
4192 }
4193
4194 partial = (dw13 >> 16) & 0x01;
4195
4196 zone_entry_sz = sizeof(NvmeZoneDescr);
4197 if (zra == NVME_ZONE_REPORT_EXTENDED) {
4198 zone_entry_sz += ns->params.zd_extension_size;
4199 }
4200
4201 max_zones = (data_size - sizeof(NvmeZoneReportHeader)) / zone_entry_sz;
4202 buf = g_malloc0(data_size);
4203
4204 zone = &ns->zone_array[zone_idx];
4205 for (i = zone_idx; i < ns->num_zones; i++) {
4206 if (partial && nr_zones >= max_zones) {
4207 break;
4208 }
4209 if (nvme_zone_matches_filter(zrasf, zone++)) {
4210 nr_zones++;
4211 }
4212 }
4213 header = buf;
4214 header->nr_zones = cpu_to_le64(nr_zones);
4215
4216 buf_p = buf + sizeof(NvmeZoneReportHeader);
4217 for (; zone_idx < ns->num_zones && max_zones > 0; zone_idx++) {
4218 zone = &ns->zone_array[zone_idx];
4219 if (nvme_zone_matches_filter(zrasf, zone)) {
4220 z = buf_p;
4221 buf_p += sizeof(NvmeZoneDescr);
4222
4223 z->zt = zone->d.zt;
4224 z->zs = zone->d.zs;
4225 z->zcap = cpu_to_le64(zone->d.zcap);
4226 z->zslba = cpu_to_le64(zone->d.zslba);
4227 z->za = zone->d.za;
4228
4229 if (nvme_wp_is_valid(zone)) {
4230 z->wp = cpu_to_le64(zone->d.wp);
4231 } else {
4232 z->wp = cpu_to_le64(~0ULL);
4233 }
4234
4235 if (zra == NVME_ZONE_REPORT_EXTENDED) {
4236 if (zone->d.za & NVME_ZA_ZD_EXT_VALID) {
4237 memcpy(buf_p, nvme_get_zd_extension(ns, zone_idx),
4238 ns->params.zd_extension_size);
4239 }
4240 buf_p += ns->params.zd_extension_size;
4241 }
4242
4243 max_zones--;
4244 }
4245 }
4246
4247 status = nvme_c2h(n, (uint8_t *)buf, data_size, req);
4248
4249 g_free(buf);
4250
4251 return status;
4252 }
4253
4254 static uint16_t nvme_io_mgmt_recv_ruhs(NvmeCtrl *n, NvmeRequest *req,
4255 size_t len)
4256 {
4257 NvmeNamespace *ns = req->ns;
4258 NvmeEnduranceGroup *endgrp;
4259 NvmeRuhStatus *hdr;
4260 NvmeRuhStatusDescr *ruhsd;
4261 unsigned int nruhsd;
4262 uint16_t rg, ph, *ruhid;
4263 size_t trans_len;
4264 g_autofree uint8_t *buf = NULL;
4265
4266 if (!n->subsys) {
4267 return NVME_INVALID_FIELD | NVME_DNR;
4268 }
4269
4270 if (ns->params.nsid == 0 || ns->params.nsid == 0xffffffff) {
4271 return NVME_INVALID_NSID | NVME_DNR;
4272 }
4273
4274 if (!n->subsys->endgrp.fdp.enabled) {
4275 return NVME_FDP_DISABLED | NVME_DNR;
4276 }
4277
4278 endgrp = ns->endgrp;
4279
4280 nruhsd = ns->fdp.nphs * endgrp->fdp.nrg;
4281 trans_len = sizeof(NvmeRuhStatus) + nruhsd * sizeof(NvmeRuhStatusDescr);
4282 buf = g_malloc(trans_len);
4283
4284 trans_len = MIN(trans_len, len);
4285
4286 hdr = (NvmeRuhStatus *)buf;
4287 ruhsd = (NvmeRuhStatusDescr *)(buf + sizeof(NvmeRuhStatus));
4288
4289 hdr->nruhsd = cpu_to_le16(nruhsd);
4290
4291 ruhid = ns->fdp.phs;
4292
4293 for (ph = 0; ph < ns->fdp.nphs; ph++, ruhid++) {
4294 NvmeRuHandle *ruh = &endgrp->fdp.ruhs[*ruhid];
4295
4296 for (rg = 0; rg < endgrp->fdp.nrg; rg++, ruhsd++) {
4297 uint16_t pid = nvme_make_pid(ns, rg, ph);
4298
4299 ruhsd->pid = cpu_to_le16(pid);
4300 ruhsd->ruhid = *ruhid;
4301 ruhsd->earutr = 0;
4302 ruhsd->ruamw = cpu_to_le64(ruh->rus[rg].ruamw);
4303 }
4304 }
4305
4306 return nvme_c2h(n, buf, trans_len, req);
4307 }
4308
4309 static uint16_t nvme_io_mgmt_recv(NvmeCtrl *n, NvmeRequest *req)
4310 {
4311 NvmeCmd *cmd = &req->cmd;
4312 uint32_t cdw10 = le32_to_cpu(cmd->cdw10);
4313 uint32_t numd = le32_to_cpu(cmd->cdw11);
4314 uint8_t mo = (cdw10 & 0xff);
4315 size_t len = (numd + 1) << 2;
4316
4317 switch (mo) {
4318 case NVME_IOMR_MO_NOP:
4319 return 0;
4320 case NVME_IOMR_MO_RUH_STATUS:
4321 return nvme_io_mgmt_recv_ruhs(n, req, len);
4322 default:
4323 return NVME_INVALID_FIELD | NVME_DNR;
4324 };
4325 }
4326
4327 static uint16_t nvme_io_mgmt_send_ruh_update(NvmeCtrl *n, NvmeRequest *req)
4328 {
4329 NvmeCmd *cmd = &req->cmd;
4330 NvmeNamespace *ns = req->ns;
4331 uint32_t cdw10 = le32_to_cpu(cmd->cdw10);
4332 uint16_t ret = NVME_SUCCESS;
4333 uint32_t npid = (cdw10 >> 1) + 1;
4334 unsigned int i = 0;
4335 g_autofree uint16_t *pids = NULL;
4336 uint32_t maxnpid = n->subsys->endgrp.fdp.nrg * n->subsys->endgrp.fdp.nruh;
4337
4338 if (unlikely(npid >= MIN(NVME_FDP_MAXPIDS, maxnpid))) {
4339 return NVME_INVALID_FIELD | NVME_DNR;
4340 }
4341
4342 pids = g_new(uint16_t, npid);
4343
4344 ret = nvme_h2c(n, pids, npid * sizeof(uint16_t), req);
4345 if (ret) {
4346 return ret;
4347 }
4348
4349 for (; i < npid; i++) {
4350 if (!nvme_update_ruh(n, ns, pids[i])) {
4351 return NVME_INVALID_FIELD | NVME_DNR;
4352 }
4353 }
4354
4355 return ret;
4356 }
4357
4358 static uint16_t nvme_io_mgmt_send(NvmeCtrl *n, NvmeRequest *req)
4359 {
4360 NvmeCmd *cmd = &req->cmd;
4361 uint32_t cdw10 = le32_to_cpu(cmd->cdw10);
4362 uint8_t mo = (cdw10 & 0xff);
4363
4364 switch (mo) {
4365 case NVME_IOMS_MO_NOP:
4366 return 0;
4367 case NVME_IOMS_MO_RUH_UPDATE:
4368 return nvme_io_mgmt_send_ruh_update(n, req);
4369 default:
4370 return NVME_INVALID_FIELD | NVME_DNR;
4371 };
4372 }
4373
4374 static uint16_t nvme_io_cmd(NvmeCtrl *n, NvmeRequest *req)
4375 {
4376 NvmeNamespace *ns;
4377 uint32_t nsid = le32_to_cpu(req->cmd.nsid);
4378
4379 trace_pci_nvme_io_cmd(nvme_cid(req), nsid, nvme_sqid(req),
4380 req->cmd.opcode, nvme_io_opc_str(req->cmd.opcode));
4381
4382 if (!nvme_nsid_valid(n, nsid)) {
4383 return NVME_INVALID_NSID | NVME_DNR;
4384 }
4385
4386 /*
4387 * In the base NVM command set, Flush may apply to all namespaces
4388 * (indicated by NSID being set to FFFFFFFFh). But if that feature is used
4389 * along with TP 4056 (Namespace Types), it may be pretty screwed up.
4390 *
4391 * If NSID is indeed set to FFFFFFFFh, we simply cannot associate the
4392 * opcode with a specific command since we cannot determine a unique I/O
4393 * command set. Opcode 0h could have any other meaning than something
4394 * equivalent to flushing and say it DOES have completely different
4395 * semantics in some other command set - does an NSID of FFFFFFFFh then
4396 * mean "for all namespaces, apply whatever command set specific command
4397 * that uses the 0h opcode?" Or does it mean "for all namespaces, apply
4398 * whatever command that uses the 0h opcode if, and only if, it allows NSID
4399 * to be FFFFFFFFh"?
4400 *
4401 * Anyway (and luckily), for now, we do not care about this since the
4402 * device only supports namespace types that includes the NVM Flush command
4403 * (NVM and Zoned), so always do an NVM Flush.
4404 */
4405 if (req->cmd.opcode == NVME_CMD_FLUSH) {
4406 return nvme_flush(n, req);
4407 }
4408
4409 ns = nvme_ns(n, nsid);
4410 if (unlikely(!ns)) {
4411 return NVME_INVALID_FIELD | NVME_DNR;
4412 }
4413
4414 if (!(ns->iocs[req->cmd.opcode] & NVME_CMD_EFF_CSUPP)) {
4415 trace_pci_nvme_err_invalid_opc(req->cmd.opcode);
4416 return NVME_INVALID_OPCODE | NVME_DNR;
4417 }
4418
4419 if (ns->status) {
4420 return ns->status;
4421 }
4422
4423 if (NVME_CMD_FLAGS_FUSE(req->cmd.flags)) {
4424 return NVME_INVALID_FIELD;
4425 }
4426
4427 req->ns = ns;
4428
4429 switch (req->cmd.opcode) {
4430 case NVME_CMD_WRITE_ZEROES:
4431 return nvme_write_zeroes(n, req);
4432 case NVME_CMD_ZONE_APPEND:
4433 return nvme_zone_append(n, req);
4434 case NVME_CMD_WRITE:
4435 return nvme_write(n, req);
4436 case NVME_CMD_READ:
4437 return nvme_read(n, req);
4438 case NVME_CMD_COMPARE:
4439 return nvme_compare(n, req);
4440 case NVME_CMD_DSM:
4441 return nvme_dsm(n, req);
4442 case NVME_CMD_VERIFY:
4443 return nvme_verify(n, req);
4444 case NVME_CMD_COPY:
4445 return nvme_copy(n, req);
4446 case NVME_CMD_ZONE_MGMT_SEND:
4447 return nvme_zone_mgmt_send(n, req);
4448 case NVME_CMD_ZONE_MGMT_RECV:
4449 return nvme_zone_mgmt_recv(n, req);
4450 case NVME_CMD_IO_MGMT_RECV:
4451 return nvme_io_mgmt_recv(n, req);
4452 case NVME_CMD_IO_MGMT_SEND:
4453 return nvme_io_mgmt_send(n, req);
4454 default:
4455 assert(false);
4456 }
4457
4458 return NVME_INVALID_OPCODE | NVME_DNR;
4459 }
4460
4461 static void nvme_cq_notifier(EventNotifier *e)
4462 {
4463 NvmeCQueue *cq = container_of(e, NvmeCQueue, notifier);
4464 NvmeCtrl *n = cq->ctrl;
4465
4466 if (!event_notifier_test_and_clear(e)) {
4467 return;
4468 }
4469
4470 nvme_update_cq_head(cq);
4471
4472 if (cq->tail == cq->head) {
4473 if (cq->irq_enabled) {
4474 n->cq_pending--;
4475 }
4476
4477 nvme_irq_deassert(n, cq);
4478 }
4479
4480 qemu_bh_schedule(cq->bh);
4481 }
4482
4483 static int nvme_init_cq_ioeventfd(NvmeCQueue *cq)
4484 {
4485 NvmeCtrl *n = cq->ctrl;
4486 uint16_t offset = (cq->cqid << 3) + (1 << 2);
4487 int ret;
4488
4489 ret = event_notifier_init(&cq->notifier, 0);
4490 if (ret < 0) {
4491 return ret;
4492 }
4493
4494 event_notifier_set_handler(&cq->notifier, nvme_cq_notifier);
4495 memory_region_add_eventfd(&n->iomem,
4496 0x1000 + offset, 4, false, 0, &cq->notifier);
4497
4498 return 0;
4499 }
4500
4501 static void nvme_sq_notifier(EventNotifier *e)
4502 {
4503 NvmeSQueue *sq = container_of(e, NvmeSQueue, notifier);
4504
4505 if (!event_notifier_test_and_clear(e)) {
4506 return;
4507 }
4508
4509 nvme_process_sq(sq);
4510 }
4511
4512 static int nvme_init_sq_ioeventfd(NvmeSQueue *sq)
4513 {
4514 NvmeCtrl *n = sq->ctrl;
4515 uint16_t offset = sq->sqid << 3;
4516 int ret;
4517
4518 ret = event_notifier_init(&sq->notifier, 0);
4519 if (ret < 0) {
4520 return ret;
4521 }
4522
4523 event_notifier_set_handler(&sq->notifier, nvme_sq_notifier);
4524 memory_region_add_eventfd(&n->iomem,
4525 0x1000 + offset, 4, false, 0, &sq->notifier);
4526
4527 return 0;
4528 }
4529
4530 static void nvme_free_sq(NvmeSQueue *sq, NvmeCtrl *n)
4531 {
4532 uint16_t offset = sq->sqid << 3;
4533
4534 n->sq[sq->sqid] = NULL;
4535 qemu_bh_delete(sq->bh);
4536 if (sq->ioeventfd_enabled) {
4537 memory_region_del_eventfd(&n->iomem,
4538 0x1000 + offset, 4, false, 0, &sq->notifier);
4539 event_notifier_set_handler(&sq->notifier, NULL);
4540 event_notifier_cleanup(&sq->notifier);
4541 }
4542 g_free(sq->io_req);
4543 if (sq->sqid) {
4544 g_free(sq);
4545 }
4546 }
4547
4548 static uint16_t nvme_del_sq(NvmeCtrl *n, NvmeRequest *req)
4549 {
4550 NvmeDeleteQ *c = (NvmeDeleteQ *)&req->cmd;
4551 NvmeRequest *r, *next;
4552 NvmeSQueue *sq;
4553 NvmeCQueue *cq;
4554 uint16_t qid = le16_to_cpu(c->qid);
4555
4556 if (unlikely(!qid || nvme_check_sqid(n, qid))) {
4557 trace_pci_nvme_err_invalid_del_sq(qid);
4558 return NVME_INVALID_QID | NVME_DNR;
4559 }
4560
4561 trace_pci_nvme_del_sq(qid);
4562
4563 sq = n->sq[qid];
4564 while (!QTAILQ_EMPTY(&sq->out_req_list)) {
4565 r = QTAILQ_FIRST(&sq->out_req_list);
4566 assert(r->aiocb);
4567 blk_aio_cancel(r->aiocb);
4568 }
4569
4570 assert(QTAILQ_EMPTY(&sq->out_req_list));
4571
4572 if (!nvme_check_cqid(n, sq->cqid)) {
4573 cq = n->cq[sq->cqid];
4574 QTAILQ_REMOVE(&cq->sq_list, sq, entry);
4575
4576 nvme_post_cqes(cq);
4577 QTAILQ_FOREACH_SAFE(r, &cq->req_list, entry, next) {
4578 if (r->sq == sq) {
4579 QTAILQ_REMOVE(&cq->req_list, r, entry);
4580 QTAILQ_INSERT_TAIL(&sq->req_list, r, entry);
4581 }
4582 }
4583 }
4584
4585 nvme_free_sq(sq, n);
4586 return NVME_SUCCESS;
4587 }
4588
4589 static void nvme_init_sq(NvmeSQueue *sq, NvmeCtrl *n, uint64_t dma_addr,
4590 uint16_t sqid, uint16_t cqid, uint16_t size)
4591 {
4592 int i;
4593 NvmeCQueue *cq;
4594
4595 sq->ctrl = n;
4596 sq->dma_addr = dma_addr;
4597 sq->sqid = sqid;
4598 sq->size = size;
4599 sq->cqid = cqid;
4600 sq->head = sq->tail = 0;
4601 sq->io_req = g_new0(NvmeRequest, sq->size);
4602
4603 QTAILQ_INIT(&sq->req_list);
4604 QTAILQ_INIT(&sq->out_req_list);
4605 for (i = 0; i < sq->size; i++) {
4606 sq->io_req[i].sq = sq;
4607 QTAILQ_INSERT_TAIL(&(sq->req_list), &sq->io_req[i], entry);
4608 }
4609
4610 sq->bh = qemu_bh_new_guarded(nvme_process_sq, sq,
4611 &DEVICE(sq->ctrl)->mem_reentrancy_guard);
4612
4613 if (n->dbbuf_enabled) {
4614 sq->db_addr = n->dbbuf_dbs + (sqid << 3);
4615 sq->ei_addr = n->dbbuf_eis + (sqid << 3);
4616
4617 if (n->params.ioeventfd && sq->sqid != 0) {
4618 if (!nvme_init_sq_ioeventfd(sq)) {
4619 sq->ioeventfd_enabled = true;
4620 }
4621 }
4622 }
4623
4624 assert(n->cq[cqid]);
4625 cq = n->cq[cqid];
4626 QTAILQ_INSERT_TAIL(&(cq->sq_list), sq, entry);
4627 n->sq[sqid] = sq;
4628 }
4629
4630 static uint16_t nvme_create_sq(NvmeCtrl *n, NvmeRequest *req)
4631 {
4632 NvmeSQueue *sq;
4633 NvmeCreateSq *c = (NvmeCreateSq *)&req->cmd;
4634
4635 uint16_t cqid = le16_to_cpu(c->cqid);
4636 uint16_t sqid = le16_to_cpu(c->sqid);
4637 uint16_t qsize = le16_to_cpu(c->qsize);
4638 uint16_t qflags = le16_to_cpu(c->sq_flags);
4639 uint64_t prp1 = le64_to_cpu(c->prp1);
4640
4641 trace_pci_nvme_create_sq(prp1, sqid, cqid, qsize, qflags);
4642
4643 if (unlikely(!cqid || nvme_check_cqid(n, cqid))) {
4644 trace_pci_nvme_err_invalid_create_sq_cqid(cqid);
4645 return NVME_INVALID_CQID | NVME_DNR;
4646 }
4647 if (unlikely(!sqid || sqid > n->conf_ioqpairs || n->sq[sqid] != NULL)) {
4648 trace_pci_nvme_err_invalid_create_sq_sqid(sqid);
4649 return NVME_INVALID_QID | NVME_DNR;
4650 }
4651 if (unlikely(!qsize || qsize > NVME_CAP_MQES(ldq_le_p(&n->bar.cap)))) {
4652 trace_pci_nvme_err_invalid_create_sq_size(qsize);
4653 return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
4654 }
4655 if (unlikely(prp1 & (n->page_size - 1))) {
4656 trace_pci_nvme_err_invalid_create_sq_addr(prp1);
4657 return NVME_INVALID_PRP_OFFSET | NVME_DNR;
4658 }
4659 if (unlikely(!(NVME_SQ_FLAGS_PC(qflags)))) {
4660 trace_pci_nvme_err_invalid_create_sq_qflags(NVME_SQ_FLAGS_PC(qflags));
4661 return NVME_INVALID_FIELD | NVME_DNR;
4662 }
4663 sq = g_malloc0(sizeof(*sq));
4664 nvme_init_sq(sq, n, prp1, sqid, cqid, qsize + 1);
4665 return NVME_SUCCESS;
4666 }
4667
4668 struct nvme_stats {
4669 uint64_t units_read;
4670 uint64_t units_written;
4671 uint64_t read_commands;
4672 uint64_t write_commands;
4673 };
4674
4675 static void nvme_set_blk_stats(NvmeNamespace *ns, struct nvme_stats *stats)
4676 {
4677 BlockAcctStats *s = blk_get_stats(ns->blkconf.blk);
4678
4679 stats->units_read += s->nr_bytes[BLOCK_ACCT_READ];
4680 stats->units_written += s->nr_bytes[BLOCK_ACCT_WRITE];
4681 stats->read_commands += s->nr_ops[BLOCK_ACCT_READ];
4682 stats->write_commands += s->nr_ops[BLOCK_ACCT_WRITE];
4683 }
4684
4685 static uint16_t nvme_smart_info(NvmeCtrl *n, uint8_t rae, uint32_t buf_len,
4686 uint64_t off, NvmeRequest *req)
4687 {
4688 uint32_t nsid = le32_to_cpu(req->cmd.nsid);
4689 struct nvme_stats stats = { 0 };
4690 NvmeSmartLog smart = { 0 };
4691 uint32_t trans_len;
4692 NvmeNamespace *ns;
4693 time_t current_ms;
4694 uint64_t u_read, u_written;
4695
4696 if (off >= sizeof(smart)) {
4697 return NVME_INVALID_FIELD | NVME_DNR;
4698 }
4699
4700 if (nsid != 0xffffffff) {
4701 ns = nvme_ns(n, nsid);
4702 if (!ns) {
4703 return NVME_INVALID_NSID | NVME_DNR;
4704 }
4705 nvme_set_blk_stats(ns, &stats);
4706 } else {
4707 int i;
4708
4709 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
4710 ns = nvme_ns(n, i);
4711 if (!ns) {
4712 continue;
4713 }
4714 nvme_set_blk_stats(ns, &stats);
4715 }
4716 }
4717
4718 trans_len = MIN(sizeof(smart) - off, buf_len);
4719 smart.critical_warning = n->smart_critical_warning;
4720
4721 u_read = DIV_ROUND_UP(stats.units_read >> BDRV_SECTOR_BITS, 1000);
4722 u_written = DIV_ROUND_UP(stats.units_written >> BDRV_SECTOR_BITS, 1000);
4723
4724 smart.data_units_read[0] = cpu_to_le64(u_read);
4725 smart.data_units_written[0] = cpu_to_le64(u_written);
4726 smart.host_read_commands[0] = cpu_to_le64(stats.read_commands);
4727 smart.host_write_commands[0] = cpu_to_le64(stats.write_commands);
4728
4729 smart.temperature = cpu_to_le16(n->temperature);
4730
4731 if ((n->temperature >= n->features.temp_thresh_hi) ||
4732 (n->temperature <= n->features.temp_thresh_low)) {
4733 smart.critical_warning |= NVME_SMART_TEMPERATURE;
4734 }
4735
4736 current_ms = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
4737 smart.power_on_hours[0] =
4738 cpu_to_le64((((current_ms - n->starttime_ms) / 1000) / 60) / 60);
4739
4740 if (!rae) {
4741 nvme_clear_events(n, NVME_AER_TYPE_SMART);
4742 }
4743
4744 return nvme_c2h(n, (uint8_t *) &smart + off, trans_len, req);
4745 }
4746
4747 static uint16_t nvme_endgrp_info(NvmeCtrl *n, uint8_t rae, uint32_t buf_len,
4748 uint64_t off, NvmeRequest *req)
4749 {
4750 uint32_t dw11 = le32_to_cpu(req->cmd.cdw11);
4751 uint16_t endgrpid = (dw11 >> 16) & 0xffff;
4752 struct nvme_stats stats = {};
4753 NvmeEndGrpLog info = {};
4754 int i;
4755
4756 if (!n->subsys || endgrpid != 0x1) {
4757 return NVME_INVALID_FIELD | NVME_DNR;
4758 }
4759
4760 if (off >= sizeof(info)) {
4761 return NVME_INVALID_FIELD | NVME_DNR;
4762 }
4763
4764 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
4765 NvmeNamespace *ns = nvme_subsys_ns(n->subsys, i);
4766 if (!ns) {
4767 continue;
4768 }
4769
4770 nvme_set_blk_stats(ns, &stats);
4771 }
4772
4773 info.data_units_read[0] =
4774 cpu_to_le64(DIV_ROUND_UP(stats.units_read / 1000000000, 1000000000));
4775 info.data_units_written[0] =
4776 cpu_to_le64(DIV_ROUND_UP(stats.units_written / 1000000000, 1000000000));
4777 info.media_units_written[0] =
4778 cpu_to_le64(DIV_ROUND_UP(stats.units_written / 1000000000, 1000000000));
4779
4780 info.host_read_commands[0] = cpu_to_le64(stats.read_commands);
4781 info.host_write_commands[0] = cpu_to_le64(stats.write_commands);
4782
4783 buf_len = MIN(sizeof(info) - off, buf_len);
4784
4785 return nvme_c2h(n, (uint8_t *)&info + off, buf_len, req);
4786 }
4787
4788
4789 static uint16_t nvme_fw_log_info(NvmeCtrl *n, uint32_t buf_len, uint64_t off,
4790 NvmeRequest *req)
4791 {
4792 uint32_t trans_len;
4793 NvmeFwSlotInfoLog fw_log = {
4794 .afi = 0x1,
4795 };
4796
4797 if (off >= sizeof(fw_log)) {
4798 return NVME_INVALID_FIELD | NVME_DNR;
4799 }
4800
4801 strpadcpy((char *)&fw_log.frs1, sizeof(fw_log.frs1), "1.0", ' ');
4802 trans_len = MIN(sizeof(fw_log) - off, buf_len);
4803
4804 return nvme_c2h(n, (uint8_t *) &fw_log + off, trans_len, req);
4805 }
4806
4807 static uint16_t nvme_error_info(NvmeCtrl *n, uint8_t rae, uint32_t buf_len,
4808 uint64_t off, NvmeRequest *req)
4809 {
4810 uint32_t trans_len;
4811 NvmeErrorLog errlog;
4812
4813 if (off >= sizeof(errlog)) {
4814 return NVME_INVALID_FIELD | NVME_DNR;
4815 }
4816
4817 if (!rae) {
4818 nvme_clear_events(n, NVME_AER_TYPE_ERROR);
4819 }
4820
4821 memset(&errlog, 0x0, sizeof(errlog));
4822 trans_len = MIN(sizeof(errlog) - off, buf_len);
4823
4824 return nvme_c2h(n, (uint8_t *)&errlog, trans_len, req);
4825 }
4826
4827 static uint16_t nvme_changed_nslist(NvmeCtrl *n, uint8_t rae, uint32_t buf_len,
4828 uint64_t off, NvmeRequest *req)
4829 {
4830 uint32_t nslist[1024];
4831 uint32_t trans_len;
4832 int i = 0;
4833 uint32_t nsid;
4834
4835 if (off >= sizeof(nslist)) {
4836 trace_pci_nvme_err_invalid_log_page_offset(off, sizeof(nslist));
4837 return NVME_INVALID_FIELD | NVME_DNR;
4838 }
4839
4840 memset(nslist, 0x0, sizeof(nslist));
4841 trans_len = MIN(sizeof(nslist) - off, buf_len);
4842
4843 while ((nsid = find_first_bit(n->changed_nsids, NVME_CHANGED_NSID_SIZE)) !=
4844 NVME_CHANGED_NSID_SIZE) {
4845 /*
4846 * If more than 1024 namespaces, the first entry in the log page should
4847 * be set to FFFFFFFFh and the others to 0 as spec.
4848 */
4849 if (i == ARRAY_SIZE(nslist)) {
4850 memset(nslist, 0x0, sizeof(nslist));
4851 nslist[0] = 0xffffffff;
4852 break;
4853 }
4854
4855 nslist[i++] = nsid;
4856 clear_bit(nsid, n->changed_nsids);
4857 }
4858
4859 /*
4860 * Remove all the remaining list entries in case returns directly due to
4861 * more than 1024 namespaces.
4862 */
4863 if (nslist[0] == 0xffffffff) {
4864 bitmap_zero(n->changed_nsids, NVME_CHANGED_NSID_SIZE);
4865 }
4866
4867 if (!rae) {
4868 nvme_clear_events(n, NVME_AER_TYPE_NOTICE);
4869 }
4870
4871 return nvme_c2h(n, ((uint8_t *)nslist) + off, trans_len, req);
4872 }
4873
4874 static uint16_t nvme_cmd_effects(NvmeCtrl *n, uint8_t csi, uint32_t buf_len,
4875 uint64_t off, NvmeRequest *req)
4876 {
4877 NvmeEffectsLog log = {};
4878 const uint32_t *src_iocs = NULL;
4879 uint32_t trans_len;
4880
4881 if (off >= sizeof(log)) {
4882 trace_pci_nvme_err_invalid_log_page_offset(off, sizeof(log));
4883 return NVME_INVALID_FIELD | NVME_DNR;
4884 }
4885
4886 switch (NVME_CC_CSS(ldl_le_p(&n->bar.cc))) {
4887 case NVME_CC_CSS_NVM:
4888 src_iocs = nvme_cse_iocs_nvm;
4889 /* fall through */
4890 case NVME_CC_CSS_ADMIN_ONLY:
4891 break;
4892 case NVME_CC_CSS_CSI:
4893 switch (csi) {
4894 case NVME_CSI_NVM:
4895 src_iocs = nvme_cse_iocs_nvm;
4896 break;
4897 case NVME_CSI_ZONED:
4898 src_iocs = nvme_cse_iocs_zoned;
4899 break;
4900 }
4901 }
4902
4903 memcpy(log.acs, nvme_cse_acs, sizeof(nvme_cse_acs));
4904
4905 if (src_iocs) {
4906 memcpy(log.iocs, src_iocs, sizeof(log.iocs));
4907 }
4908
4909 trans_len = MIN(sizeof(log) - off, buf_len);
4910
4911 return nvme_c2h(n, ((uint8_t *)&log) + off, trans_len, req);
4912 }
4913
4914 static size_t sizeof_fdp_conf_descr(size_t nruh, size_t vss)
4915 {
4916 size_t entry_siz = sizeof(NvmeFdpDescrHdr) + nruh * sizeof(NvmeRuhDescr)
4917 + vss;
4918 return ROUND_UP(entry_siz, 8);
4919 }
4920
4921 static uint16_t nvme_fdp_confs(NvmeCtrl *n, uint32_t endgrpid, uint32_t buf_len,
4922 uint64_t off, NvmeRequest *req)
4923 {
4924 uint32_t log_size, trans_len;
4925 g_autofree uint8_t *buf = NULL;
4926 NvmeFdpDescrHdr *hdr;
4927 NvmeRuhDescr *ruhd;
4928 NvmeEnduranceGroup *endgrp;
4929 NvmeFdpConfsHdr *log;
4930 size_t nruh, fdp_descr_size;
4931 int i;
4932
4933 if (endgrpid != 1 || !n->subsys) {
4934 return NVME_INVALID_FIELD | NVME_DNR;
4935 }
4936
4937 endgrp = &n->subsys->endgrp;
4938
4939 if (endgrp->fdp.enabled) {
4940 nruh = endgrp->fdp.nruh;
4941 } else {
4942 nruh = 1;
4943 }
4944
4945 fdp_descr_size = sizeof_fdp_conf_descr(nruh, FDPVSS);
4946 log_size = sizeof(NvmeFdpConfsHdr) + fdp_descr_size;
4947
4948 if (off >= log_size) {
4949 return NVME_INVALID_FIELD | NVME_DNR;
4950 }
4951
4952 trans_len = MIN(log_size - off, buf_len);
4953
4954 buf = g_malloc0(log_size);
4955 log = (NvmeFdpConfsHdr *)buf;
4956 hdr = (NvmeFdpDescrHdr *)(log + 1);
4957 ruhd = (NvmeRuhDescr *)(buf + sizeof(*log) + sizeof(*hdr));
4958
4959 log->num_confs = cpu_to_le16(0);
4960 log->size = cpu_to_le32(log_size);
4961
4962 hdr->descr_size = cpu_to_le16(fdp_descr_size);
4963 if (endgrp->fdp.enabled) {
4964 hdr->fdpa = FIELD_DP8(hdr->fdpa, FDPA, VALID, 1);
4965 hdr->fdpa = FIELD_DP8(hdr->fdpa, FDPA, RGIF, endgrp->fdp.rgif);
4966 hdr->nrg = cpu_to_le16(endgrp->fdp.nrg);
4967 hdr->nruh = cpu_to_le16(endgrp->fdp.nruh);
4968 hdr->maxpids = cpu_to_le16(NVME_FDP_MAXPIDS - 1);
4969 hdr->nnss = cpu_to_le32(NVME_MAX_NAMESPACES);
4970 hdr->runs = cpu_to_le64(endgrp->fdp.runs);
4971
4972 for (i = 0; i < nruh; i++) {
4973 ruhd->ruht = NVME_RUHT_INITIALLY_ISOLATED;
4974 ruhd++;
4975 }
4976 } else {
4977 /* 1 bit for RUH in PIF -> 2 RUHs max. */
4978 hdr->nrg = cpu_to_le16(1);
4979 hdr->nruh = cpu_to_le16(1);
4980 hdr->maxpids = cpu_to_le16(NVME_FDP_MAXPIDS - 1);
4981 hdr->nnss = cpu_to_le32(1);
4982 hdr->runs = cpu_to_le64(96 * MiB);
4983
4984 ruhd->ruht = NVME_RUHT_INITIALLY_ISOLATED;
4985 }
4986
4987 return nvme_c2h(n, (uint8_t *)buf + off, trans_len, req);
4988 }
4989
4990 static uint16_t nvme_fdp_ruh_usage(NvmeCtrl *n, uint32_t endgrpid,
4991 uint32_t dw10, uint32_t dw12,
4992 uint32_t buf_len, uint64_t off,
4993 NvmeRequest *req)
4994 {
4995 NvmeRuHandle *ruh;
4996 NvmeRuhuLog *hdr;
4997 NvmeRuhuDescr *ruhud;
4998 NvmeEnduranceGroup *endgrp;
4999 g_autofree uint8_t *buf = NULL;
5000 uint32_t log_size, trans_len;
5001 uint16_t i;
5002
5003 if (endgrpid != 1 || !n->subsys) {
5004 return NVME_INVALID_FIELD | NVME_DNR;
5005 }
5006
5007 endgrp = &n->subsys->endgrp;
5008
5009 if (!endgrp->fdp.enabled) {
5010 return NVME_FDP_DISABLED | NVME_DNR;
5011 }
5012
5013 log_size = sizeof(NvmeRuhuLog) + endgrp->fdp.nruh * sizeof(NvmeRuhuDescr);
5014
5015 if (off >= log_size) {
5016 return NVME_INVALID_FIELD | NVME_DNR;
5017 }
5018
5019 trans_len = MIN(log_size - off, buf_len);
5020
5021 buf = g_malloc0(log_size);
5022 hdr = (NvmeRuhuLog *)buf;
5023 ruhud = (NvmeRuhuDescr *)(hdr + 1);
5024
5025 ruh = endgrp->fdp.ruhs;
5026 hdr->nruh = cpu_to_le16(endgrp->fdp.nruh);
5027
5028 for (i = 0; i < endgrp->fdp.nruh; i++, ruhud++, ruh++) {
5029 ruhud->ruha = ruh->ruha;
5030 }
5031
5032 return nvme_c2h(n, (uint8_t *)buf + off, trans_len, req);
5033 }
5034
5035 static uint16_t nvme_fdp_stats(NvmeCtrl *n, uint32_t endgrpid, uint32_t buf_len,
5036 uint64_t off, NvmeRequest *req)
5037 {
5038 NvmeEnduranceGroup *endgrp;
5039 NvmeFdpStatsLog log = {};
5040 uint32_t trans_len;
5041
5042 if (off >= sizeof(NvmeFdpStatsLog)) {
5043 return NVME_INVALID_FIELD | NVME_DNR;
5044 }
5045
5046 if (endgrpid != 1 || !n->subsys) {
5047 return NVME_INVALID_FIELD | NVME_DNR;
5048 }
5049
5050 if (!n->subsys->endgrp.fdp.enabled) {
5051 return NVME_FDP_DISABLED | NVME_DNR;
5052 }
5053
5054 endgrp = &n->subsys->endgrp;
5055
5056 trans_len = MIN(sizeof(log) - off, buf_len);
5057
5058 /* spec value is 128 bit, we only use 64 bit */
5059 log.hbmw[0] = cpu_to_le64(endgrp->fdp.hbmw);
5060 log.mbmw[0] = cpu_to_le64(endgrp->fdp.mbmw);
5061 log.mbe[0] = cpu_to_le64(endgrp->fdp.mbe);
5062
5063 return nvme_c2h(n, (uint8_t *)&log + off, trans_len, req);
5064 }
5065
5066 static uint16_t nvme_fdp_events(NvmeCtrl *n, uint32_t endgrpid,
5067 uint32_t buf_len, uint64_t off,
5068 NvmeRequest *req)
5069 {
5070 NvmeEnduranceGroup *endgrp;
5071 NvmeCmd *cmd = &req->cmd;
5072 bool host_events = (cmd->cdw10 >> 8) & 0x1;
5073 uint32_t log_size, trans_len;
5074 NvmeFdpEventBuffer *ebuf;
5075 g_autofree NvmeFdpEventsLog *elog = NULL;
5076 NvmeFdpEvent *event;
5077
5078 if (endgrpid != 1 || !n->subsys) {
5079 return NVME_INVALID_FIELD | NVME_DNR;
5080 }
5081
5082 endgrp = &n->subsys->endgrp;
5083
5084 if (!endgrp->fdp.enabled) {
5085 return NVME_FDP_DISABLED | NVME_DNR;
5086 }
5087
5088 if (host_events) {
5089 ebuf = &endgrp->fdp.host_events;
5090 } else {
5091 ebuf = &endgrp->fdp.ctrl_events;
5092 }
5093
5094 log_size = sizeof(NvmeFdpEventsLog) + ebuf->nelems * sizeof(NvmeFdpEvent);
5095 trans_len = MIN(log_size - off, buf_len);
5096 elog = g_malloc0(log_size);
5097 elog->num_events = cpu_to_le32(ebuf->nelems);
5098 event = (NvmeFdpEvent *)(elog + 1);
5099
5100 if (ebuf->nelems && ebuf->start == ebuf->next) {
5101 unsigned int nelems = (NVME_FDP_MAX_EVENTS - ebuf->start);
5102 /* wrap over, copy [start;NVME_FDP_MAX_EVENTS[ and [0; next[ */
5103 memcpy(event, &ebuf->events[ebuf->start],
5104 sizeof(NvmeFdpEvent) * nelems);
5105 memcpy(event + nelems, ebuf->events,
5106 sizeof(NvmeFdpEvent) * ebuf->next);
5107 } else if (ebuf->start < ebuf->next) {
5108 memcpy(event, &ebuf->events[ebuf->start],
5109 sizeof(NvmeFdpEvent) * (ebuf->next - ebuf->start));
5110 }
5111
5112 return nvme_c2h(n, (uint8_t *)elog + off, trans_len, req);
5113 }
5114
5115 static uint16_t nvme_get_log(NvmeCtrl *n, NvmeRequest *req)
5116 {
5117 NvmeCmd *cmd = &req->cmd;
5118
5119 uint32_t dw10 = le32_to_cpu(cmd->cdw10);
5120 uint32_t dw11 = le32_to_cpu(cmd->cdw11);
5121 uint32_t dw12 = le32_to_cpu(cmd->cdw12);
5122 uint32_t dw13 = le32_to_cpu(cmd->cdw13);
5123 uint8_t lid = dw10 & 0xff;
5124 uint8_t lsp = (dw10 >> 8) & 0xf;
5125 uint8_t rae = (dw10 >> 15) & 0x1;
5126 uint8_t csi = le32_to_cpu(cmd->cdw14) >> 24;
5127 uint32_t numdl, numdu, lspi;
5128 uint64_t off, lpol, lpou;
5129 size_t len;
5130 uint16_t status;
5131
5132 numdl = (dw10 >> 16);
5133 numdu = (dw11 & 0xffff);
5134 lspi = (dw11 >> 16);
5135 lpol = dw12;
5136 lpou = dw13;
5137
5138 len = (((numdu << 16) | numdl) + 1) << 2;
5139 off = (lpou << 32ULL) | lpol;
5140
5141 if (off & 0x3) {
5142 return NVME_INVALID_FIELD | NVME_DNR;
5143 }
5144
5145 trace_pci_nvme_get_log(nvme_cid(req), lid, lsp, rae, len, off);
5146
5147 status = nvme_check_mdts(n, len);
5148 if (status) {
5149 return status;
5150 }
5151
5152 switch (lid) {
5153 case NVME_LOG_ERROR_INFO:
5154 return nvme_error_info(n, rae, len, off, req);
5155 case NVME_LOG_SMART_INFO:
5156 return nvme_smart_info(n, rae, len, off, req);
5157 case NVME_LOG_FW_SLOT_INFO:
5158 return nvme_fw_log_info(n, len, off, req);
5159 case NVME_LOG_CHANGED_NSLIST:
5160 return nvme_changed_nslist(n, rae, len, off, req);
5161 case NVME_LOG_CMD_EFFECTS:
5162 return nvme_cmd_effects(n, csi, len, off, req);
5163 case NVME_LOG_ENDGRP:
5164 return nvme_endgrp_info(n, rae, len, off, req);
5165 case NVME_LOG_FDP_CONFS:
5166 return nvme_fdp_confs(n, lspi, len, off, req);
5167 case NVME_LOG_FDP_RUH_USAGE:
5168 return nvme_fdp_ruh_usage(n, lspi, dw10, dw12, len, off, req);
5169 case NVME_LOG_FDP_STATS:
5170 return nvme_fdp_stats(n, lspi, len, off, req);
5171 case NVME_LOG_FDP_EVENTS:
5172 return nvme_fdp_events(n, lspi, len, off, req);
5173 default:
5174 trace_pci_nvme_err_invalid_log_page(nvme_cid(req), lid);
5175 return NVME_INVALID_FIELD | NVME_DNR;
5176 }
5177 }
5178
5179 static void nvme_free_cq(NvmeCQueue *cq, NvmeCtrl *n)
5180 {
5181 PCIDevice *pci = PCI_DEVICE(n);
5182 uint16_t offset = (cq->cqid << 3) + (1 << 2);
5183
5184 n->cq[cq->cqid] = NULL;
5185 qemu_bh_delete(cq->bh);
5186 if (cq->ioeventfd_enabled) {
5187 memory_region_del_eventfd(&n->iomem,
5188 0x1000 + offset, 4, false, 0, &cq->notifier);
5189 event_notifier_set_handler(&cq->notifier, NULL);
5190 event_notifier_cleanup(&cq->notifier);
5191 }
5192 if (msix_enabled(pci)) {
5193 msix_vector_unuse(pci, cq->vector);
5194 }
5195 if (cq->cqid) {
5196 g_free(cq);
5197 }
5198 }
5199
5200 static uint16_t nvme_del_cq(NvmeCtrl *n, NvmeRequest *req)
5201 {
5202 NvmeDeleteQ *c = (NvmeDeleteQ *)&req->cmd;
5203 NvmeCQueue *cq;
5204 uint16_t qid = le16_to_cpu(c->qid);
5205
5206 if (unlikely(!qid || nvme_check_cqid(n, qid))) {
5207 trace_pci_nvme_err_invalid_del_cq_cqid(qid);
5208 return NVME_INVALID_CQID | NVME_DNR;
5209 }
5210
5211 cq = n->cq[qid];
5212 if (unlikely(!QTAILQ_EMPTY(&cq->sq_list))) {
5213 trace_pci_nvme_err_invalid_del_cq_notempty(qid);
5214 return NVME_INVALID_QUEUE_DEL;
5215 }
5216
5217 if (cq->irq_enabled && cq->tail != cq->head) {
5218 n->cq_pending--;
5219 }
5220
5221 nvme_irq_deassert(n, cq);
5222 trace_pci_nvme_del_cq(qid);
5223 nvme_free_cq(cq, n);
5224 return NVME_SUCCESS;
5225 }
5226
5227 static void nvme_init_cq(NvmeCQueue *cq, NvmeCtrl *n, uint64_t dma_addr,
5228 uint16_t cqid, uint16_t vector, uint16_t size,
5229 uint16_t irq_enabled)
5230 {
5231 PCIDevice *pci = PCI_DEVICE(n);
5232
5233 if (msix_enabled(pci)) {
5234 msix_vector_use(pci, vector);
5235 }
5236 cq->ctrl = n;
5237 cq->cqid = cqid;
5238 cq->size = size;
5239 cq->dma_addr = dma_addr;
5240 cq->phase = 1;
5241 cq->irq_enabled = irq_enabled;
5242 cq->vector = vector;
5243 cq->head = cq->tail = 0;
5244 QTAILQ_INIT(&cq->req_list);
5245 QTAILQ_INIT(&cq->sq_list);
5246 if (n->dbbuf_enabled) {
5247 cq->db_addr = n->dbbuf_dbs + (cqid << 3) + (1 << 2);
5248 cq->ei_addr = n->dbbuf_eis + (cqid << 3) + (1 << 2);
5249
5250 if (n->params.ioeventfd && cqid != 0) {
5251 if (!nvme_init_cq_ioeventfd(cq)) {
5252 cq->ioeventfd_enabled = true;
5253 }
5254 }
5255 }
5256 n->cq[cqid] = cq;
5257 cq->bh = qemu_bh_new_guarded(nvme_post_cqes, cq,
5258 &DEVICE(cq->ctrl)->mem_reentrancy_guard);
5259 }
5260
5261 static uint16_t nvme_create_cq(NvmeCtrl *n, NvmeRequest *req)
5262 {
5263 NvmeCQueue *cq;
5264 NvmeCreateCq *c = (NvmeCreateCq *)&req->cmd;
5265 uint16_t cqid = le16_to_cpu(c->cqid);
5266 uint16_t vector = le16_to_cpu(c->irq_vector);
5267 uint16_t qsize = le16_to_cpu(c->qsize);
5268 uint16_t qflags = le16_to_cpu(c->cq_flags);
5269 uint64_t prp1 = le64_to_cpu(c->prp1);
5270
5271 trace_pci_nvme_create_cq(prp1, cqid, vector, qsize, qflags,
5272 NVME_CQ_FLAGS_IEN(qflags) != 0);
5273
5274 if (unlikely(!cqid || cqid > n->conf_ioqpairs || n->cq[cqid] != NULL)) {
5275 trace_pci_nvme_err_invalid_create_cq_cqid(cqid);
5276 return NVME_INVALID_QID | NVME_DNR;
5277 }
5278 if (unlikely(!qsize || qsize > NVME_CAP_MQES(ldq_le_p(&n->bar.cap)))) {
5279 trace_pci_nvme_err_invalid_create_cq_size(qsize);
5280 return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
5281 }
5282 if (unlikely(prp1 & (n->page_size - 1))) {
5283 trace_pci_nvme_err_invalid_create_cq_addr(prp1);
5284 return NVME_INVALID_PRP_OFFSET | NVME_DNR;
5285 }
5286 if (unlikely(!msix_enabled(PCI_DEVICE(n)) && vector)) {
5287 trace_pci_nvme_err_invalid_create_cq_vector(vector);
5288 return NVME_INVALID_IRQ_VECTOR | NVME_DNR;
5289 }
5290 if (unlikely(vector >= n->conf_msix_qsize)) {
5291 trace_pci_nvme_err_invalid_create_cq_vector(vector);
5292 return NVME_INVALID_IRQ_VECTOR | NVME_DNR;
5293 }
5294 if (unlikely(!(NVME_CQ_FLAGS_PC(qflags)))) {
5295 trace_pci_nvme_err_invalid_create_cq_qflags(NVME_CQ_FLAGS_PC(qflags));
5296 return NVME_INVALID_FIELD | NVME_DNR;
5297 }
5298
5299 cq = g_malloc0(sizeof(*cq));
5300 nvme_init_cq(cq, n, prp1, cqid, vector, qsize + 1,
5301 NVME_CQ_FLAGS_IEN(qflags));
5302
5303 /*
5304 * It is only required to set qs_created when creating a completion queue;
5305 * creating a submission queue without a matching completion queue will
5306 * fail.
5307 */
5308 n->qs_created = true;
5309 return NVME_SUCCESS;
5310 }
5311
5312 static uint16_t nvme_rpt_empty_id_struct(NvmeCtrl *n, NvmeRequest *req)
5313 {
5314 uint8_t id[NVME_IDENTIFY_DATA_SIZE] = {};
5315
5316 return nvme_c2h(n, id, sizeof(id), req);
5317 }
5318
5319 static uint16_t nvme_identify_ctrl(NvmeCtrl *n, NvmeRequest *req)
5320 {
5321 trace_pci_nvme_identify_ctrl();
5322
5323 return nvme_c2h(n, (uint8_t *)&n->id_ctrl, sizeof(n->id_ctrl), req);
5324 }
5325
5326 static uint16_t nvme_identify_ctrl_csi(NvmeCtrl *n, NvmeRequest *req)
5327 {
5328 NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5329 uint8_t id[NVME_IDENTIFY_DATA_SIZE] = {};
5330 NvmeIdCtrlNvm *id_nvm = (NvmeIdCtrlNvm *)&id;
5331
5332 trace_pci_nvme_identify_ctrl_csi(c->csi);
5333
5334 switch (c->csi) {
5335 case NVME_CSI_NVM:
5336 id_nvm->vsl = n->params.vsl;
5337 id_nvm->dmrsl = cpu_to_le32(n->dmrsl);
5338 break;
5339
5340 case NVME_CSI_ZONED:
5341 ((NvmeIdCtrlZoned *)&id)->zasl = n->params.zasl;
5342 break;
5343
5344 default:
5345 return NVME_INVALID_FIELD | NVME_DNR;
5346 }
5347
5348 return nvme_c2h(n, id, sizeof(id), req);
5349 }
5350
5351 static uint16_t nvme_identify_ns(NvmeCtrl *n, NvmeRequest *req, bool active)
5352 {
5353 NvmeNamespace *ns;
5354 NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5355 uint32_t nsid = le32_to_cpu(c->nsid);
5356
5357 trace_pci_nvme_identify_ns(nsid);
5358
5359 if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
5360 return NVME_INVALID_NSID | NVME_DNR;
5361 }
5362
5363 ns = nvme_ns(n, nsid);
5364 if (unlikely(!ns)) {
5365 if (!active) {
5366 ns = nvme_subsys_ns(n->subsys, nsid);
5367 if (!ns) {
5368 return nvme_rpt_empty_id_struct(n, req);
5369 }
5370 } else {
5371 return nvme_rpt_empty_id_struct(n, req);
5372 }
5373 }
5374
5375 if (active || ns->csi == NVME_CSI_NVM) {
5376 return nvme_c2h(n, (uint8_t *)&ns->id_ns, sizeof(NvmeIdNs), req);
5377 }
5378
5379 return NVME_INVALID_CMD_SET | NVME_DNR;
5380 }
5381
5382 static uint16_t nvme_identify_ctrl_list(NvmeCtrl *n, NvmeRequest *req,
5383 bool attached)
5384 {
5385 NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5386 uint32_t nsid = le32_to_cpu(c->nsid);
5387 uint16_t min_id = le16_to_cpu(c->ctrlid);
5388 uint16_t list[NVME_CONTROLLER_LIST_SIZE] = {};
5389 uint16_t *ids = &list[1];
5390 NvmeNamespace *ns;
5391 NvmeCtrl *ctrl;
5392 int cntlid, nr_ids = 0;
5393
5394 trace_pci_nvme_identify_ctrl_list(c->cns, min_id);
5395
5396 if (!n->subsys) {
5397 return NVME_INVALID_FIELD | NVME_DNR;
5398 }
5399
5400 if (attached) {
5401 if (nsid == NVME_NSID_BROADCAST) {
5402 return NVME_INVALID_FIELD | NVME_DNR;
5403 }
5404
5405 ns = nvme_subsys_ns(n->subsys, nsid);
5406 if (!ns) {
5407 return NVME_INVALID_FIELD | NVME_DNR;
5408 }
5409 }
5410
5411 for (cntlid = min_id; cntlid < ARRAY_SIZE(n->subsys->ctrls); cntlid++) {
5412 ctrl = nvme_subsys_ctrl(n->subsys, cntlid);
5413 if (!ctrl) {
5414 continue;
5415 }
5416
5417 if (attached && !nvme_ns(ctrl, nsid)) {
5418 continue;
5419 }
5420
5421 ids[nr_ids++] = cntlid;
5422 }
5423
5424 list[0] = nr_ids;
5425
5426 return nvme_c2h(n, (uint8_t *)list, sizeof(list), req);
5427 }
5428
5429 static uint16_t nvme_identify_pri_ctrl_cap(NvmeCtrl *n, NvmeRequest *req)
5430 {
5431 trace_pci_nvme_identify_pri_ctrl_cap(le16_to_cpu(n->pri_ctrl_cap.cntlid));
5432
5433 return nvme_c2h(n, (uint8_t *)&n->pri_ctrl_cap,
5434 sizeof(NvmePriCtrlCap), req);
5435 }
5436
5437 static uint16_t nvme_identify_sec_ctrl_list(NvmeCtrl *n, NvmeRequest *req)
5438 {
5439 NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5440 uint16_t pri_ctrl_id = le16_to_cpu(n->pri_ctrl_cap.cntlid);
5441 uint16_t min_id = le16_to_cpu(c->ctrlid);
5442 uint8_t num_sec_ctrl = n->sec_ctrl_list.numcntl;
5443 NvmeSecCtrlList list = {0};
5444 uint8_t i;
5445
5446 for (i = 0; i < num_sec_ctrl; i++) {
5447 if (n->sec_ctrl_list.sec[i].scid >= min_id) {
5448 list.numcntl = num_sec_ctrl - i;
5449 memcpy(&list.sec, n->sec_ctrl_list.sec + i,
5450 list.numcntl * sizeof(NvmeSecCtrlEntry));
5451 break;
5452 }
5453 }
5454
5455 trace_pci_nvme_identify_sec_ctrl_list(pri_ctrl_id, list.numcntl);
5456
5457 return nvme_c2h(n, (uint8_t *)&list, sizeof(list), req);
5458 }
5459
5460 static uint16_t nvme_identify_ns_csi(NvmeCtrl *n, NvmeRequest *req,
5461 bool active)
5462 {
5463 NvmeNamespace *ns;
5464 NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5465 uint32_t nsid = le32_to_cpu(c->nsid);
5466
5467 trace_pci_nvme_identify_ns_csi(nsid, c->csi);
5468
5469 if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
5470 return NVME_INVALID_NSID | NVME_DNR;
5471 }
5472
5473 ns = nvme_ns(n, nsid);
5474 if (unlikely(!ns)) {
5475 if (!active) {
5476 ns = nvme_subsys_ns(n->subsys, nsid);
5477 if (!ns) {
5478 return nvme_rpt_empty_id_struct(n, req);
5479 }
5480 } else {
5481 return nvme_rpt_empty_id_struct(n, req);
5482 }
5483 }
5484
5485 if (c->csi == NVME_CSI_NVM) {
5486 return nvme_c2h(n, (uint8_t *)&ns->id_ns_nvm, sizeof(NvmeIdNsNvm),
5487 req);
5488 } else if (c->csi == NVME_CSI_ZONED && ns->csi == NVME_CSI_ZONED) {
5489 return nvme_c2h(n, (uint8_t *)ns->id_ns_zoned, sizeof(NvmeIdNsZoned),
5490 req);
5491 }
5492
5493 return NVME_INVALID_FIELD | NVME_DNR;
5494 }
5495
5496 static uint16_t nvme_identify_nslist(NvmeCtrl *n, NvmeRequest *req,
5497 bool active)
5498 {
5499 NvmeNamespace *ns;
5500 NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5501 uint32_t min_nsid = le32_to_cpu(c->nsid);
5502 uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {};
5503 static const int data_len = sizeof(list);
5504 uint32_t *list_ptr = (uint32_t *)list;
5505 int i, j = 0;
5506
5507 trace_pci_nvme_identify_nslist(min_nsid);
5508
5509 /*
5510 * Both FFFFFFFFh (NVME_NSID_BROADCAST) and FFFFFFFFEh are invalid values
5511 * since the Active Namespace ID List should return namespaces with ids
5512 * *higher* than the NSID specified in the command. This is also specified
5513 * in the spec (NVM Express v1.3d, Section 5.15.4).
5514 */
5515 if (min_nsid >= NVME_NSID_BROADCAST - 1) {
5516 return NVME_INVALID_NSID | NVME_DNR;
5517 }
5518
5519 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
5520 ns = nvme_ns(n, i);
5521 if (!ns) {
5522 if (!active) {
5523 ns = nvme_subsys_ns(n->subsys, i);
5524 if (!ns) {
5525 continue;
5526 }
5527 } else {
5528 continue;
5529 }
5530 }
5531 if (ns->params.nsid <= min_nsid) {
5532 continue;
5533 }
5534 list_ptr[j++] = cpu_to_le32(ns->params.nsid);
5535 if (j == data_len / sizeof(uint32_t)) {
5536 break;
5537 }
5538 }
5539
5540 return nvme_c2h(n, list, data_len, req);
5541 }
5542
5543 static uint16_t nvme_identify_nslist_csi(NvmeCtrl *n, NvmeRequest *req,
5544 bool active)
5545 {
5546 NvmeNamespace *ns;
5547 NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5548 uint32_t min_nsid = le32_to_cpu(c->nsid);
5549 uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {};
5550 static const int data_len = sizeof(list);
5551 uint32_t *list_ptr = (uint32_t *)list;
5552 int i, j = 0;
5553
5554 trace_pci_nvme_identify_nslist_csi(min_nsid, c->csi);
5555
5556 /*
5557 * Same as in nvme_identify_nslist(), FFFFFFFFh/FFFFFFFFEh are invalid.
5558 */
5559 if (min_nsid >= NVME_NSID_BROADCAST - 1) {
5560 return NVME_INVALID_NSID | NVME_DNR;
5561 }
5562
5563 if (c->csi != NVME_CSI_NVM && c->csi != NVME_CSI_ZONED) {
5564 return NVME_INVALID_FIELD | NVME_DNR;
5565 }
5566
5567 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
5568 ns = nvme_ns(n, i);
5569 if (!ns) {
5570 if (!active) {
5571 ns = nvme_subsys_ns(n->subsys, i);
5572 if (!ns) {
5573 continue;
5574 }
5575 } else {
5576 continue;
5577 }
5578 }
5579 if (ns->params.nsid <= min_nsid || c->csi != ns->csi) {
5580 continue;
5581 }
5582 list_ptr[j++] = cpu_to_le32(ns->params.nsid);
5583 if (j == data_len / sizeof(uint32_t)) {
5584 break;
5585 }
5586 }
5587
5588 return nvme_c2h(n, list, data_len, req);
5589 }
5590
5591 static uint16_t nvme_identify_ns_descr_list(NvmeCtrl *n, NvmeRequest *req)
5592 {
5593 NvmeNamespace *ns;
5594 NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5595 uint32_t nsid = le32_to_cpu(c->nsid);
5596 uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {};
5597 uint8_t *pos = list;
5598 struct {
5599 NvmeIdNsDescr hdr;
5600 uint8_t v[NVME_NIDL_UUID];
5601 } QEMU_PACKED uuid = {};
5602 struct {
5603 NvmeIdNsDescr hdr;
5604 uint64_t v;
5605 } QEMU_PACKED eui64 = {};
5606 struct {
5607 NvmeIdNsDescr hdr;
5608 uint8_t v;
5609 } QEMU_PACKED csi = {};
5610
5611 trace_pci_nvme_identify_ns_descr_list(nsid);
5612
5613 if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
5614 return NVME_INVALID_NSID | NVME_DNR;
5615 }
5616
5617 ns = nvme_ns(n, nsid);
5618 if (unlikely(!ns)) {
5619 return NVME_INVALID_FIELD | NVME_DNR;
5620 }
5621
5622 if (!qemu_uuid_is_null(&ns->params.uuid)) {
5623 uuid.hdr.nidt = NVME_NIDT_UUID;
5624 uuid.hdr.nidl = NVME_NIDL_UUID;
5625 memcpy(uuid.v, ns->params.uuid.data, NVME_NIDL_UUID);
5626 memcpy(pos, &uuid, sizeof(uuid));
5627 pos += sizeof(uuid);
5628 }
5629
5630 if (ns->params.eui64) {
5631 eui64.hdr.nidt = NVME_NIDT_EUI64;
5632 eui64.hdr.nidl = NVME_NIDL_EUI64;
5633 eui64.v = cpu_to_be64(ns->params.eui64);
5634 memcpy(pos, &eui64, sizeof(eui64));
5635 pos += sizeof(eui64);
5636 }
5637
5638 csi.hdr.nidt = NVME_NIDT_CSI;
5639 csi.hdr.nidl = NVME_NIDL_CSI;
5640 csi.v = ns->csi;
5641 memcpy(pos, &csi, sizeof(csi));
5642 pos += sizeof(csi);
5643
5644 return nvme_c2h(n, list, sizeof(list), req);
5645 }
5646
5647 static uint16_t nvme_identify_cmd_set(NvmeCtrl *n, NvmeRequest *req)
5648 {
5649 uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {};
5650 static const int data_len = sizeof(list);
5651
5652 trace_pci_nvme_identify_cmd_set();
5653
5654 NVME_SET_CSI(*list, NVME_CSI_NVM);
5655 NVME_SET_CSI(*list, NVME_CSI_ZONED);
5656
5657 return nvme_c2h(n, list, data_len, req);
5658 }
5659
5660 static uint16_t nvme_identify(NvmeCtrl *n, NvmeRequest *req)
5661 {
5662 NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5663
5664 trace_pci_nvme_identify(nvme_cid(req), c->cns, le16_to_cpu(c->ctrlid),
5665 c->csi);
5666
5667 switch (c->cns) {
5668 case NVME_ID_CNS_NS:
5669 return nvme_identify_ns(n, req, true);
5670 case NVME_ID_CNS_NS_PRESENT:
5671 return nvme_identify_ns(n, req, false);
5672 case NVME_ID_CNS_NS_ATTACHED_CTRL_LIST:
5673 return nvme_identify_ctrl_list(n, req, true);
5674 case NVME_ID_CNS_CTRL_LIST:
5675 return nvme_identify_ctrl_list(n, req, false);
5676 case NVME_ID_CNS_PRIMARY_CTRL_CAP:
5677 return nvme_identify_pri_ctrl_cap(n, req);
5678 case NVME_ID_CNS_SECONDARY_CTRL_LIST:
5679 return nvme_identify_sec_ctrl_list(n, req);
5680 case NVME_ID_CNS_CS_NS:
5681 return nvme_identify_ns_csi(n, req, true);
5682 case NVME_ID_CNS_CS_NS_PRESENT:
5683 return nvme_identify_ns_csi(n, req, false);
5684 case NVME_ID_CNS_CTRL:
5685 return nvme_identify_ctrl(n, req);
5686 case NVME_ID_CNS_CS_CTRL:
5687 return nvme_identify_ctrl_csi(n, req);
5688 case NVME_ID_CNS_NS_ACTIVE_LIST:
5689 return nvme_identify_nslist(n, req, true);
5690 case NVME_ID_CNS_NS_PRESENT_LIST:
5691 return nvme_identify_nslist(n, req, false);
5692 case NVME_ID_CNS_CS_NS_ACTIVE_LIST:
5693 return nvme_identify_nslist_csi(n, req, true);
5694 case NVME_ID_CNS_CS_NS_PRESENT_LIST:
5695 return nvme_identify_nslist_csi(n, req, false);
5696 case NVME_ID_CNS_NS_DESCR_LIST:
5697 return nvme_identify_ns_descr_list(n, req);
5698 case NVME_ID_CNS_IO_COMMAND_SET:
5699 return nvme_identify_cmd_set(n, req);
5700 default:
5701 trace_pci_nvme_err_invalid_identify_cns(le32_to_cpu(c->cns));
5702 return NVME_INVALID_FIELD | NVME_DNR;
5703 }
5704 }
5705
5706 static uint16_t nvme_abort(NvmeCtrl *n, NvmeRequest *req)
5707 {
5708 uint16_t sqid = le32_to_cpu(req->cmd.cdw10) & 0xffff;
5709
5710 req->cqe.result = 1;
5711 if (nvme_check_sqid(n, sqid)) {
5712 return NVME_INVALID_FIELD | NVME_DNR;
5713 }
5714
5715 return NVME_SUCCESS;
5716 }
5717
5718 static inline void nvme_set_timestamp(NvmeCtrl *n, uint64_t ts)
5719 {
5720 trace_pci_nvme_setfeat_timestamp(ts);
5721
5722 n->host_timestamp = le64_to_cpu(ts);
5723 n->timestamp_set_qemu_clock_ms = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
5724 }
5725
5726 static inline uint64_t nvme_get_timestamp(const NvmeCtrl *n)
5727 {
5728 uint64_t current_time = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
5729 uint64_t elapsed_time = current_time - n->timestamp_set_qemu_clock_ms;
5730
5731 union nvme_timestamp {
5732 struct {
5733 uint64_t timestamp:48;
5734 uint64_t sync:1;
5735 uint64_t origin:3;
5736 uint64_t rsvd1:12;
5737 };
5738 uint64_t all;
5739 };
5740
5741 union nvme_timestamp ts;
5742 ts.all = 0;
5743 ts.timestamp = n->host_timestamp + elapsed_time;
5744
5745 /* If the host timestamp is non-zero, set the timestamp origin */
5746 ts.origin = n->host_timestamp ? 0x01 : 0x00;
5747
5748 trace_pci_nvme_getfeat_timestamp(ts.all);
5749
5750 return cpu_to_le64(ts.all);
5751 }
5752
5753 static uint16_t nvme_get_feature_timestamp(NvmeCtrl *n, NvmeRequest *req)
5754 {
5755 uint64_t timestamp = nvme_get_timestamp(n);
5756
5757 return nvme_c2h(n, (uint8_t *)&timestamp, sizeof(timestamp), req);
5758 }
5759
5760 static int nvme_get_feature_fdp(NvmeCtrl *n, uint32_t endgrpid,
5761 uint32_t *result)
5762 {
5763 *result = 0;
5764
5765 if (!n->subsys || !n->subsys->endgrp.fdp.enabled) {
5766 return NVME_INVALID_FIELD | NVME_DNR;
5767 }
5768
5769 *result = FIELD_DP16(0, FEAT_FDP, FDPE, 1);
5770 *result = FIELD_DP16(*result, FEAT_FDP, CONF_NDX, 0);
5771
5772 return NVME_SUCCESS;
5773 }
5774
5775 static uint16_t nvme_get_feature_fdp_events(NvmeCtrl *n, NvmeNamespace *ns,
5776 NvmeRequest *req, uint32_t *result)
5777 {
5778 NvmeCmd *cmd = &req->cmd;
5779 uint32_t cdw11 = le32_to_cpu(cmd->cdw11);
5780 uint16_t ph = cdw11 & 0xffff;
5781 uint8_t noet = (cdw11 >> 16) & 0xff;
5782 uint16_t ruhid, ret;
5783 uint32_t nentries = 0;
5784 uint8_t s_events_ndx = 0;
5785 size_t s_events_siz = sizeof(NvmeFdpEventDescr) * noet;
5786 g_autofree NvmeFdpEventDescr *s_events = g_malloc0(s_events_siz);
5787 NvmeRuHandle *ruh;
5788 NvmeFdpEventDescr *s_event;
5789
5790 if (!n->subsys || !n->subsys->endgrp.fdp.enabled) {
5791 return NVME_FDP_DISABLED | NVME_DNR;
5792 }
5793
5794 if (!nvme_ph_valid(ns, ph)) {
5795 return NVME_INVALID_FIELD | NVME_DNR;
5796 }
5797
5798 ruhid = ns->fdp.phs[ph];
5799 ruh = &n->subsys->endgrp.fdp.ruhs[ruhid];
5800
5801 assert(ruh);
5802
5803 if (unlikely(noet == 0)) {
5804 return NVME_INVALID_FIELD | NVME_DNR;
5805 }
5806
5807 for (uint8_t event_type = 0; event_type < FDP_EVT_MAX; event_type++) {
5808 uint8_t shift = nvme_fdp_evf_shifts[event_type];
5809 if (!shift && event_type) {
5810 /*
5811 * only first entry (event_type == 0) has a shift value of 0
5812 * other entries are simply unpopulated.
5813 */
5814 continue;
5815 }
5816
5817 nentries++;
5818
5819 s_event = &s_events[s_events_ndx];
5820 s_event->evt = event_type;
5821 s_event->evta = (ruh->event_filter >> shift) & 0x1;
5822
5823 /* break if all `noet` entries are filled */
5824 if ((++s_events_ndx) == noet) {
5825 break;
5826 }
5827 }
5828
5829 ret = nvme_c2h(n, s_events, s_events_siz, req);
5830 if (ret) {
5831 return ret;
5832 }
5833
5834 *result = nentries;
5835 return NVME_SUCCESS;
5836 }
5837
5838 static uint16_t nvme_get_feature(NvmeCtrl *n, NvmeRequest *req)
5839 {
5840 NvmeCmd *cmd = &req->cmd;
5841 uint32_t dw10 = le32_to_cpu(cmd->cdw10);
5842 uint32_t dw11 = le32_to_cpu(cmd->cdw11);
5843 uint32_t nsid = le32_to_cpu(cmd->nsid);
5844 uint32_t result;
5845 uint8_t fid = NVME_GETSETFEAT_FID(dw10);
5846 NvmeGetFeatureSelect sel = NVME_GETFEAT_SELECT(dw10);
5847 uint16_t iv;
5848 NvmeNamespace *ns;
5849 int i;
5850 uint16_t endgrpid = 0, ret = NVME_SUCCESS;
5851
5852 static const uint32_t nvme_feature_default[NVME_FID_MAX] = {
5853 [NVME_ARBITRATION] = NVME_ARB_AB_NOLIMIT,
5854 };
5855
5856 trace_pci_nvme_getfeat(nvme_cid(req), nsid, fid, sel, dw11);
5857
5858 if (!nvme_feature_support[fid]) {
5859 return NVME_INVALID_FIELD | NVME_DNR;
5860 }
5861
5862 if (nvme_feature_cap[fid] & NVME_FEAT_CAP_NS) {
5863 if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
5864 /*
5865 * The Reservation Notification Mask and Reservation Persistence
5866 * features require a status code of Invalid Field in Command when
5867 * NSID is FFFFFFFFh. Since the device does not support those
5868 * features we can always return Invalid Namespace or Format as we
5869 * should do for all other features.
5870 */
5871 return NVME_INVALID_NSID | NVME_DNR;
5872 }
5873
5874 if (!nvme_ns(n, nsid)) {
5875 return NVME_INVALID_FIELD | NVME_DNR;
5876 }
5877 }
5878
5879 switch (sel) {
5880 case NVME_GETFEAT_SELECT_CURRENT:
5881 break;
5882 case NVME_GETFEAT_SELECT_SAVED:
5883 /* no features are saveable by the controller; fallthrough */
5884 case NVME_GETFEAT_SELECT_DEFAULT:
5885 goto defaults;
5886 case NVME_GETFEAT_SELECT_CAP:
5887 result = nvme_feature_cap[fid];
5888 goto out;
5889 }
5890
5891 switch (fid) {
5892 case NVME_TEMPERATURE_THRESHOLD:
5893 result = 0;
5894
5895 /*
5896 * The controller only implements the Composite Temperature sensor, so
5897 * return 0 for all other sensors.
5898 */
5899 if (NVME_TEMP_TMPSEL(dw11) != NVME_TEMP_TMPSEL_COMPOSITE) {
5900 goto out;
5901 }
5902
5903 switch (NVME_TEMP_THSEL(dw11)) {
5904 case NVME_TEMP_THSEL_OVER:
5905 result = n->features.temp_thresh_hi;
5906 goto out;
5907 case NVME_TEMP_THSEL_UNDER:
5908 result = n->features.temp_thresh_low;
5909 goto out;
5910 }
5911
5912 return NVME_INVALID_FIELD | NVME_DNR;
5913 case NVME_ERROR_RECOVERY:
5914 if (!nvme_nsid_valid(n, nsid)) {
5915 return NVME_INVALID_NSID | NVME_DNR;
5916 }
5917
5918 ns = nvme_ns(n, nsid);
5919 if (unlikely(!ns)) {
5920 return NVME_INVALID_FIELD | NVME_DNR;
5921 }
5922
5923 result = ns->features.err_rec;
5924 goto out;
5925 case NVME_VOLATILE_WRITE_CACHE:
5926 result = 0;
5927 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
5928 ns = nvme_ns(n, i);
5929 if (!ns) {
5930 continue;
5931 }
5932
5933 result = blk_enable_write_cache(ns->blkconf.blk);
5934 if (result) {
5935 break;
5936 }
5937 }
5938 trace_pci_nvme_getfeat_vwcache(result ? "enabled" : "disabled");
5939 goto out;
5940 case NVME_ASYNCHRONOUS_EVENT_CONF:
5941 result = n->features.async_config;
5942 goto out;
5943 case NVME_TIMESTAMP:
5944 return nvme_get_feature_timestamp(n, req);
5945 case NVME_HOST_BEHAVIOR_SUPPORT:
5946 return nvme_c2h(n, (uint8_t *)&n->features.hbs,
5947 sizeof(n->features.hbs), req);
5948 case NVME_FDP_MODE:
5949 endgrpid = dw11 & 0xff;
5950
5951 if (endgrpid != 0x1) {
5952 return NVME_INVALID_FIELD | NVME_DNR;
5953 }
5954
5955 ret = nvme_get_feature_fdp(n, endgrpid, &result);
5956 if (ret) {
5957 return ret;
5958 }
5959 goto out;
5960 case NVME_FDP_EVENTS:
5961 if (!nvme_nsid_valid(n, nsid)) {
5962 return NVME_INVALID_NSID | NVME_DNR;
5963 }
5964
5965 ns = nvme_ns(n, nsid);
5966 if (unlikely(!ns)) {
5967 return NVME_INVALID_FIELD | NVME_DNR;
5968 }
5969
5970 ret = nvme_get_feature_fdp_events(n, ns, req, &result);
5971 if (ret) {
5972 return ret;
5973 }
5974 goto out;
5975 default:
5976 break;
5977 }
5978
5979 defaults:
5980 switch (fid) {
5981 case NVME_TEMPERATURE_THRESHOLD:
5982 result = 0;
5983
5984 if (NVME_TEMP_TMPSEL(dw11) != NVME_TEMP_TMPSEL_COMPOSITE) {
5985 break;
5986 }
5987
5988 if (NVME_TEMP_THSEL(dw11) == NVME_TEMP_THSEL_OVER) {
5989 result = NVME_TEMPERATURE_WARNING;
5990 }
5991
5992 break;
5993 case NVME_NUMBER_OF_QUEUES:
5994 result = (n->conf_ioqpairs - 1) | ((n->conf_ioqpairs - 1) << 16);
5995 trace_pci_nvme_getfeat_numq(result);
5996 break;
5997 case NVME_INTERRUPT_VECTOR_CONF:
5998 iv = dw11 & 0xffff;
5999 if (iv >= n->conf_ioqpairs + 1) {
6000 return NVME_INVALID_FIELD | NVME_DNR;
6001 }
6002
6003 result = iv;
6004 if (iv == n->admin_cq.vector) {
6005 result |= NVME_INTVC_NOCOALESCING;
6006 }
6007 break;
6008 case NVME_FDP_MODE:
6009 endgrpid = dw11 & 0xff;
6010
6011 if (endgrpid != 0x1) {
6012 return NVME_INVALID_FIELD | NVME_DNR;
6013 }
6014
6015 ret = nvme_get_feature_fdp(n, endgrpid, &result);
6016 if (ret) {
6017 return ret;
6018 }
6019 goto out;
6020
6021 break;
6022 default:
6023 result = nvme_feature_default[fid];
6024 break;
6025 }
6026
6027 out:
6028 req->cqe.result = cpu_to_le32(result);
6029 return ret;
6030 }
6031
6032 static uint16_t nvme_set_feature_timestamp(NvmeCtrl *n, NvmeRequest *req)
6033 {
6034 uint16_t ret;
6035 uint64_t timestamp;
6036
6037 ret = nvme_h2c(n, (uint8_t *)&timestamp, sizeof(timestamp), req);
6038 if (ret) {
6039 return ret;
6040 }
6041
6042 nvme_set_timestamp(n, timestamp);
6043
6044 return NVME_SUCCESS;
6045 }
6046
6047 static uint16_t nvme_set_feature_fdp_events(NvmeCtrl *n, NvmeNamespace *ns,
6048 NvmeRequest *req)
6049 {
6050 NvmeCmd *cmd = &req->cmd;
6051 uint32_t cdw11 = le32_to_cpu(cmd->cdw11);
6052 uint16_t ph = cdw11 & 0xffff;
6053 uint8_t noet = (cdw11 >> 16) & 0xff;
6054 uint16_t ret, ruhid;
6055 uint8_t enable = le32_to_cpu(cmd->cdw12) & 0x1;
6056 uint8_t event_mask = 0;
6057 unsigned int i;
6058 g_autofree uint8_t *events = g_malloc0(noet);
6059 NvmeRuHandle *ruh = NULL;
6060
6061 assert(ns);
6062
6063 if (!n->subsys || !n->subsys->endgrp.fdp.enabled) {
6064 return NVME_FDP_DISABLED | NVME_DNR;
6065 }
6066
6067 if (!nvme_ph_valid(ns, ph)) {
6068 return NVME_INVALID_FIELD | NVME_DNR;
6069 }
6070
6071 ruhid = ns->fdp.phs[ph];
6072 ruh = &n->subsys->endgrp.fdp.ruhs[ruhid];
6073
6074 ret = nvme_h2c(n, events, noet, req);
6075 if (ret) {
6076 return ret;
6077 }
6078
6079 for (i = 0; i < noet; i++) {
6080 event_mask |= (1 << nvme_fdp_evf_shifts[events[i]]);
6081 }
6082
6083 if (enable) {
6084 ruh->event_filter |= event_mask;
6085 } else {
6086 ruh->event_filter = ruh->event_filter & ~event_mask;
6087 }
6088
6089 return NVME_SUCCESS;
6090 }
6091
6092 static uint16_t nvme_set_feature(NvmeCtrl *n, NvmeRequest *req)
6093 {
6094 NvmeNamespace *ns = NULL;
6095
6096 NvmeCmd *cmd = &req->cmd;
6097 uint32_t dw10 = le32_to_cpu(cmd->cdw10);
6098 uint32_t dw11 = le32_to_cpu(cmd->cdw11);
6099 uint32_t nsid = le32_to_cpu(cmd->nsid);
6100 uint8_t fid = NVME_GETSETFEAT_FID(dw10);
6101 uint8_t save = NVME_SETFEAT_SAVE(dw10);
6102 uint16_t status;
6103 int i;
6104
6105 trace_pci_nvme_setfeat(nvme_cid(req), nsid, fid, save, dw11);
6106
6107 if (save && !(nvme_feature_cap[fid] & NVME_FEAT_CAP_SAVE)) {
6108 return NVME_FID_NOT_SAVEABLE | NVME_DNR;
6109 }
6110
6111 if (!nvme_feature_support[fid]) {
6112 return NVME_INVALID_FIELD | NVME_DNR;
6113 }
6114
6115 if (nvme_feature_cap[fid] & NVME_FEAT_CAP_NS) {
6116 if (nsid != NVME_NSID_BROADCAST) {
6117 if (!nvme_nsid_valid(n, nsid)) {
6118 return NVME_INVALID_NSID | NVME_DNR;
6119 }
6120
6121 ns = nvme_ns(n, nsid);
6122 if (unlikely(!ns)) {
6123 return NVME_INVALID_FIELD | NVME_DNR;
6124 }
6125 }
6126 } else if (nsid && nsid != NVME_NSID_BROADCAST) {
6127 if (!nvme_nsid_valid(n, nsid)) {
6128 return NVME_INVALID_NSID | NVME_DNR;
6129 }
6130
6131 return NVME_FEAT_NOT_NS_SPEC | NVME_DNR;
6132 }
6133
6134 if (!(nvme_feature_cap[fid] & NVME_FEAT_CAP_CHANGE)) {
6135 return NVME_FEAT_NOT_CHANGEABLE | NVME_DNR;
6136 }
6137
6138 switch (fid) {
6139 case NVME_TEMPERATURE_THRESHOLD:
6140 if (NVME_TEMP_TMPSEL(dw11) != NVME_TEMP_TMPSEL_COMPOSITE) {
6141 break;
6142 }
6143
6144 switch (NVME_TEMP_THSEL(dw11)) {
6145 case NVME_TEMP_THSEL_OVER:
6146 n->features.temp_thresh_hi = NVME_TEMP_TMPTH(dw11);
6147 break;
6148 case NVME_TEMP_THSEL_UNDER:
6149 n->features.temp_thresh_low = NVME_TEMP_TMPTH(dw11);
6150 break;
6151 default:
6152 return NVME_INVALID_FIELD | NVME_DNR;
6153 }
6154
6155 if ((n->temperature >= n->features.temp_thresh_hi) ||
6156 (n->temperature <= n->features.temp_thresh_low)) {
6157 nvme_smart_event(n, NVME_SMART_TEMPERATURE);
6158 }
6159
6160 break;
6161 case NVME_ERROR_RECOVERY:
6162 if (nsid == NVME_NSID_BROADCAST) {
6163 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
6164 ns = nvme_ns(n, i);
6165
6166 if (!ns) {
6167 continue;
6168 }
6169
6170 if (NVME_ID_NS_NSFEAT_DULBE(ns->id_ns.nsfeat)) {
6171 ns->features.err_rec = dw11;
6172 }
6173 }
6174
6175 break;
6176 }
6177
6178 assert(ns);
6179 if (NVME_ID_NS_NSFEAT_DULBE(ns->id_ns.nsfeat)) {
6180 ns->features.err_rec = dw11;
6181 }
6182 break;
6183 case NVME_VOLATILE_WRITE_CACHE:
6184 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
6185 ns = nvme_ns(n, i);
6186 if (!ns) {
6187 continue;
6188 }
6189
6190 if (!(dw11 & 0x1) && blk_enable_write_cache(ns->blkconf.blk)) {
6191 blk_flush(ns->blkconf.blk);
6192 }
6193
6194 blk_set_enable_write_cache(ns->blkconf.blk, dw11 & 1);
6195 }
6196
6197 break;
6198
6199 case NVME_NUMBER_OF_QUEUES:
6200 if (n->qs_created) {
6201 return NVME_CMD_SEQ_ERROR | NVME_DNR;
6202 }
6203
6204 /*
6205 * NVMe v1.3, Section 5.21.1.7: FFFFh is not an allowed value for NCQR
6206 * and NSQR.
6207 */
6208 if ((dw11 & 0xffff) == 0xffff || ((dw11 >> 16) & 0xffff) == 0xffff) {
6209 return NVME_INVALID_FIELD | NVME_DNR;
6210 }
6211
6212 trace_pci_nvme_setfeat_numq((dw11 & 0xffff) + 1,
6213 ((dw11 >> 16) & 0xffff) + 1,
6214 n->conf_ioqpairs,
6215 n->conf_ioqpairs);
6216 req->cqe.result = cpu_to_le32((n->conf_ioqpairs - 1) |
6217 ((n->conf_ioqpairs - 1) << 16));
6218 break;
6219 case NVME_ASYNCHRONOUS_EVENT_CONF:
6220 n->features.async_config = dw11;
6221 break;
6222 case NVME_TIMESTAMP:
6223 return nvme_set_feature_timestamp(n, req);
6224 case NVME_HOST_BEHAVIOR_SUPPORT:
6225 status = nvme_h2c(n, (uint8_t *)&n->features.hbs,
6226 sizeof(n->features.hbs), req);
6227 if (status) {
6228 return status;
6229 }
6230
6231 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
6232 ns = nvme_ns(n, i);
6233
6234 if (!ns) {
6235 continue;
6236 }
6237
6238 ns->id_ns.nlbaf = ns->nlbaf - 1;
6239 if (!n->features.hbs.lbafee) {
6240 ns->id_ns.nlbaf = MIN(ns->id_ns.nlbaf, 15);
6241 }
6242 }
6243
6244 return status;
6245 case NVME_COMMAND_SET_PROFILE:
6246 if (dw11 & 0x1ff) {
6247 trace_pci_nvme_err_invalid_iocsci(dw11 & 0x1ff);
6248 return NVME_CMD_SET_CMB_REJECTED | NVME_DNR;
6249 }
6250 break;
6251 case NVME_FDP_MODE:
6252 /* spec: abort with cmd seq err if there's one or more NS' in endgrp */
6253 return NVME_CMD_SEQ_ERROR | NVME_DNR;
6254 case NVME_FDP_EVENTS:
6255 return nvme_set_feature_fdp_events(n, ns, req);
6256 default:
6257 return NVME_FEAT_NOT_CHANGEABLE | NVME_DNR;
6258 }
6259 return NVME_SUCCESS;
6260 }
6261
6262 static uint16_t nvme_aer(NvmeCtrl *n, NvmeRequest *req)
6263 {
6264 trace_pci_nvme_aer(nvme_cid(req));
6265
6266 if (n->outstanding_aers > n->params.aerl) {
6267 trace_pci_nvme_aer_aerl_exceeded();
6268 return NVME_AER_LIMIT_EXCEEDED;
6269 }
6270
6271 n->aer_reqs[n->outstanding_aers] = req;
6272 n->outstanding_aers++;
6273
6274 if (!QTAILQ_EMPTY(&n->aer_queue)) {
6275 nvme_process_aers(n);
6276 }
6277
6278 return NVME_NO_COMPLETE;
6279 }
6280
6281 static void nvme_update_dmrsl(NvmeCtrl *n)
6282 {
6283 int nsid;
6284
6285 for (nsid = 1; nsid <= NVME_MAX_NAMESPACES; nsid++) {
6286 NvmeNamespace *ns = nvme_ns(n, nsid);
6287 if (!ns) {
6288 continue;
6289 }
6290
6291 n->dmrsl = MIN_NON_ZERO(n->dmrsl,
6292 BDRV_REQUEST_MAX_BYTES / nvme_l2b(ns, 1));
6293 }
6294 }
6295
6296 static void nvme_select_iocs_ns(NvmeCtrl *n, NvmeNamespace *ns)
6297 {
6298 uint32_t cc = ldl_le_p(&n->bar.cc);
6299
6300 ns->iocs = nvme_cse_iocs_none;
6301 switch (ns->csi) {
6302 case NVME_CSI_NVM:
6303 if (NVME_CC_CSS(cc) != NVME_CC_CSS_ADMIN_ONLY) {
6304 ns->iocs = nvme_cse_iocs_nvm;
6305 }
6306 break;
6307 case NVME_CSI_ZONED:
6308 if (NVME_CC_CSS(cc) == NVME_CC_CSS_CSI) {
6309 ns->iocs = nvme_cse_iocs_zoned;
6310 } else if (NVME_CC_CSS(cc) == NVME_CC_CSS_NVM) {
6311 ns->iocs = nvme_cse_iocs_nvm;
6312 }
6313 break;
6314 }
6315 }
6316
6317 static uint16_t nvme_ns_attachment(NvmeCtrl *n, NvmeRequest *req)
6318 {
6319 NvmeNamespace *ns;
6320 NvmeCtrl *ctrl;
6321 uint16_t list[NVME_CONTROLLER_LIST_SIZE] = {};
6322 uint32_t nsid = le32_to_cpu(req->cmd.nsid);
6323 uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
6324 uint8_t sel = dw10 & 0xf;
6325 uint16_t *nr_ids = &list[0];
6326 uint16_t *ids = &list[1];
6327 uint16_t ret;
6328 int i;
6329
6330 trace_pci_nvme_ns_attachment(nvme_cid(req), dw10 & 0xf);
6331
6332 if (!nvme_nsid_valid(n, nsid)) {
6333 return NVME_INVALID_NSID | NVME_DNR;
6334 }
6335
6336 ns = nvme_subsys_ns(n->subsys, nsid);
6337 if (!ns) {
6338 return NVME_INVALID_FIELD | NVME_DNR;
6339 }
6340
6341 ret = nvme_h2c(n, (uint8_t *)list, 4096, req);
6342 if (ret) {
6343 return ret;
6344 }
6345
6346 if (!*nr_ids) {
6347 return NVME_NS_CTRL_LIST_INVALID | NVME_DNR;
6348 }
6349
6350 *nr_ids = MIN(*nr_ids, NVME_CONTROLLER_LIST_SIZE - 1);
6351 for (i = 0; i < *nr_ids; i++) {
6352 ctrl = nvme_subsys_ctrl(n->subsys, ids[i]);
6353 if (!ctrl) {
6354 return NVME_NS_CTRL_LIST_INVALID | NVME_DNR;
6355 }
6356
6357 switch (sel) {
6358 case NVME_NS_ATTACHMENT_ATTACH:
6359 if (nvme_ns(ctrl, nsid)) {
6360 return NVME_NS_ALREADY_ATTACHED | NVME_DNR;
6361 }
6362
6363 if (ns->attached && !ns->params.shared) {
6364 return NVME_NS_PRIVATE | NVME_DNR;
6365 }
6366
6367 nvme_attach_ns(ctrl, ns);
6368 nvme_select_iocs_ns(ctrl, ns);
6369
6370 break;
6371
6372 case NVME_NS_ATTACHMENT_DETACH:
6373 if (!nvme_ns(ctrl, nsid)) {
6374 return NVME_NS_NOT_ATTACHED | NVME_DNR;
6375 }
6376
6377 ctrl->namespaces[nsid] = NULL;
6378 ns->attached--;
6379
6380 nvme_update_dmrsl(ctrl);
6381
6382 break;
6383
6384 default:
6385 return NVME_INVALID_FIELD | NVME_DNR;
6386 }
6387
6388 /*
6389 * Add namespace id to the changed namespace id list for event clearing
6390 * via Get Log Page command.
6391 */
6392 if (!test_and_set_bit(nsid, ctrl->changed_nsids)) {
6393 nvme_enqueue_event(ctrl, NVME_AER_TYPE_NOTICE,
6394 NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED,
6395 NVME_LOG_CHANGED_NSLIST);
6396 }
6397 }
6398
6399 return NVME_SUCCESS;
6400 }
6401
6402 typedef struct NvmeFormatAIOCB {
6403 BlockAIOCB common;
6404 BlockAIOCB *aiocb;
6405 NvmeRequest *req;
6406 int ret;
6407
6408 NvmeNamespace *ns;
6409 uint32_t nsid;
6410 bool broadcast;
6411 int64_t offset;
6412
6413 uint8_t lbaf;
6414 uint8_t mset;
6415 uint8_t pi;
6416 uint8_t pil;
6417 } NvmeFormatAIOCB;
6418
6419 static void nvme_format_cancel(BlockAIOCB *aiocb)
6420 {
6421 NvmeFormatAIOCB *iocb = container_of(aiocb, NvmeFormatAIOCB, common);
6422
6423 iocb->ret = -ECANCELED;
6424
6425 if (iocb->aiocb) {
6426 blk_aio_cancel_async(iocb->aiocb);
6427 iocb->aiocb = NULL;
6428 }
6429 }
6430
6431 static const AIOCBInfo nvme_format_aiocb_info = {
6432 .aiocb_size = sizeof(NvmeFormatAIOCB),
6433 .cancel_async = nvme_format_cancel,
6434 .get_aio_context = nvme_get_aio_context,
6435 };
6436
6437 static void nvme_format_set(NvmeNamespace *ns, uint8_t lbaf, uint8_t mset,
6438 uint8_t pi, uint8_t pil)
6439 {
6440 uint8_t lbafl = lbaf & 0xf;
6441 uint8_t lbafu = lbaf >> 4;
6442
6443 trace_pci_nvme_format_set(ns->params.nsid, lbaf, mset, pi, pil);
6444
6445 ns->id_ns.dps = (pil << 3) | pi;
6446 ns->id_ns.flbas = (lbafu << 5) | (mset << 4) | lbafl;
6447
6448 nvme_ns_init_format(ns);
6449 }
6450
6451 static void nvme_do_format(NvmeFormatAIOCB *iocb);
6452
6453 static void nvme_format_ns_cb(void *opaque, int ret)
6454 {
6455 NvmeFormatAIOCB *iocb = opaque;
6456 NvmeNamespace *ns = iocb->ns;
6457 int bytes;
6458
6459 if (iocb->ret < 0) {
6460 goto done;
6461 } else if (ret < 0) {
6462 iocb->ret = ret;
6463 goto done;
6464 }
6465
6466 assert(ns);
6467
6468 if (iocb->offset < ns->size) {
6469 bytes = MIN(BDRV_REQUEST_MAX_BYTES, ns->size - iocb->offset);
6470
6471 iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk, iocb->offset,
6472 bytes, BDRV_REQ_MAY_UNMAP,
6473 nvme_format_ns_cb, iocb);
6474
6475 iocb->offset += bytes;
6476 return;
6477 }
6478
6479 nvme_format_set(ns, iocb->lbaf, iocb->mset, iocb->pi, iocb->pil);
6480 ns->status = 0x0;
6481 iocb->ns = NULL;
6482 iocb->offset = 0;
6483
6484 done:
6485 nvme_do_format(iocb);
6486 }
6487
6488 static uint16_t nvme_format_check(NvmeNamespace *ns, uint8_t lbaf, uint8_t pi)
6489 {
6490 if (ns->params.zoned) {
6491 return NVME_INVALID_FORMAT | NVME_DNR;
6492 }
6493
6494 if (lbaf > ns->id_ns.nlbaf) {
6495 return NVME_INVALID_FORMAT | NVME_DNR;
6496 }
6497
6498 if (pi && (ns->id_ns.lbaf[lbaf].ms < nvme_pi_tuple_size(ns))) {
6499 return NVME_INVALID_FORMAT | NVME_DNR;
6500 }
6501
6502 if (pi && pi > NVME_ID_NS_DPS_TYPE_3) {
6503 return NVME_INVALID_FIELD | NVME_DNR;
6504 }
6505
6506 return NVME_SUCCESS;
6507 }
6508
6509 static void nvme_do_format(NvmeFormatAIOCB *iocb)
6510 {
6511 NvmeRequest *req = iocb->req;
6512 NvmeCtrl *n = nvme_ctrl(req);
6513 uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
6514 uint8_t lbaf = dw10 & 0xf;
6515 uint8_t pi = (dw10 >> 5) & 0x7;
6516 uint16_t status;
6517 int i;
6518
6519 if (iocb->ret < 0) {
6520 goto done;
6521 }
6522
6523 if (iocb->broadcast) {
6524 for (i = iocb->nsid + 1; i <= NVME_MAX_NAMESPACES; i++) {
6525 iocb->ns = nvme_ns(n, i);
6526 if (iocb->ns) {
6527 iocb->nsid = i;
6528 break;
6529 }
6530 }
6531 }
6532
6533 if (!iocb->ns) {
6534 goto done;
6535 }
6536
6537 status = nvme_format_check(iocb->ns, lbaf, pi);
6538 if (status) {
6539 req->status = status;
6540 goto done;
6541 }
6542
6543 iocb->ns->status = NVME_FORMAT_IN_PROGRESS;
6544 nvme_format_ns_cb(iocb, 0);
6545 return;
6546
6547 done:
6548 iocb->common.cb(iocb->common.opaque, iocb->ret);
6549 qemu_aio_unref(iocb);
6550 }
6551
6552 static uint16_t nvme_format(NvmeCtrl *n, NvmeRequest *req)
6553 {
6554 NvmeFormatAIOCB *iocb;
6555 uint32_t nsid = le32_to_cpu(req->cmd.nsid);
6556 uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
6557 uint8_t lbaf = dw10 & 0xf;
6558 uint8_t mset = (dw10 >> 4) & 0x1;
6559 uint8_t pi = (dw10 >> 5) & 0x7;
6560 uint8_t pil = (dw10 >> 8) & 0x1;
6561 uint8_t lbafu = (dw10 >> 12) & 0x3;
6562 uint16_t status;
6563
6564 iocb = qemu_aio_get(&nvme_format_aiocb_info, NULL, nvme_misc_cb, req);
6565
6566 iocb->req = req;
6567 iocb->ret = 0;
6568 iocb->ns = NULL;
6569 iocb->nsid = 0;
6570 iocb->lbaf = lbaf;
6571 iocb->mset = mset;
6572 iocb->pi = pi;
6573 iocb->pil = pil;
6574 iocb->broadcast = (nsid == NVME_NSID_BROADCAST);
6575 iocb->offset = 0;
6576
6577 if (n->features.hbs.lbafee) {
6578 iocb->lbaf |= lbafu << 4;
6579 }
6580
6581 if (!iocb->broadcast) {
6582 if (!nvme_nsid_valid(n, nsid)) {
6583 status = NVME_INVALID_NSID | NVME_DNR;
6584 goto out;
6585 }
6586
6587 iocb->ns = nvme_ns(n, nsid);
6588 if (!iocb->ns) {
6589 status = NVME_INVALID_FIELD | NVME_DNR;
6590 goto out;
6591 }
6592 }
6593
6594 req->aiocb = &iocb->common;
6595 nvme_do_format(iocb);
6596
6597 return NVME_NO_COMPLETE;
6598
6599 out:
6600 qemu_aio_unref(iocb);
6601
6602 return status;
6603 }
6604
6605 static void nvme_get_virt_res_num(NvmeCtrl *n, uint8_t rt, int *num_total,
6606 int *num_prim, int *num_sec)
6607 {
6608 *num_total = le32_to_cpu(rt ?
6609 n->pri_ctrl_cap.vifrt : n->pri_ctrl_cap.vqfrt);
6610 *num_prim = le16_to_cpu(rt ?
6611 n->pri_ctrl_cap.virfap : n->pri_ctrl_cap.vqrfap);
6612 *num_sec = le16_to_cpu(rt ? n->pri_ctrl_cap.virfa : n->pri_ctrl_cap.vqrfa);
6613 }
6614
6615 static uint16_t nvme_assign_virt_res_to_prim(NvmeCtrl *n, NvmeRequest *req,
6616 uint16_t cntlid, uint8_t rt,
6617 int nr)
6618 {
6619 int num_total, num_prim, num_sec;
6620
6621 if (cntlid != n->cntlid) {
6622 return NVME_INVALID_CTRL_ID | NVME_DNR;
6623 }
6624
6625 nvme_get_virt_res_num(n, rt, &num_total, &num_prim, &num_sec);
6626
6627 if (nr > num_total) {
6628 return NVME_INVALID_NUM_RESOURCES | NVME_DNR;
6629 }
6630
6631 if (nr > num_total - num_sec) {
6632 return NVME_INVALID_RESOURCE_ID | NVME_DNR;
6633 }
6634
6635 if (rt) {
6636 n->next_pri_ctrl_cap.virfap = cpu_to_le16(nr);
6637 } else {
6638 n->next_pri_ctrl_cap.vqrfap = cpu_to_le16(nr);
6639 }
6640
6641 req->cqe.result = cpu_to_le32(nr);
6642 return req->status;
6643 }
6644
6645 static void nvme_update_virt_res(NvmeCtrl *n, NvmeSecCtrlEntry *sctrl,
6646 uint8_t rt, int nr)
6647 {
6648 int prev_nr, prev_total;
6649
6650 if (rt) {
6651 prev_nr = le16_to_cpu(sctrl->nvi);
6652 prev_total = le32_to_cpu(n->pri_ctrl_cap.virfa);
6653 sctrl->nvi = cpu_to_le16(nr);
6654 n->pri_ctrl_cap.virfa = cpu_to_le32(prev_total + nr - prev_nr);
6655 } else {
6656 prev_nr = le16_to_cpu(sctrl->nvq);
6657 prev_total = le32_to_cpu(n->pri_ctrl_cap.vqrfa);
6658 sctrl->nvq = cpu_to_le16(nr);
6659 n->pri_ctrl_cap.vqrfa = cpu_to_le32(prev_total + nr - prev_nr);
6660 }
6661 }
6662
6663 static uint16_t nvme_assign_virt_res_to_sec(NvmeCtrl *n, NvmeRequest *req,
6664 uint16_t cntlid, uint8_t rt, int nr)
6665 {
6666 int num_total, num_prim, num_sec, num_free, diff, limit;
6667 NvmeSecCtrlEntry *sctrl;
6668
6669 sctrl = nvme_sctrl_for_cntlid(n, cntlid);
6670 if (!sctrl) {
6671 return NVME_INVALID_CTRL_ID | NVME_DNR;
6672 }
6673
6674 if (sctrl->scs) {
6675 return NVME_INVALID_SEC_CTRL_STATE | NVME_DNR;
6676 }
6677
6678 limit = le16_to_cpu(rt ? n->pri_ctrl_cap.vifrsm : n->pri_ctrl_cap.vqfrsm);
6679 if (nr > limit) {
6680 return NVME_INVALID_NUM_RESOURCES | NVME_DNR;
6681 }
6682
6683 nvme_get_virt_res_num(n, rt, &num_total, &num_prim, &num_sec);
6684 num_free = num_total - num_prim - num_sec;
6685 diff = nr - le16_to_cpu(rt ? sctrl->nvi : sctrl->nvq);
6686
6687 if (diff > num_free) {
6688 return NVME_INVALID_RESOURCE_ID | NVME_DNR;
6689 }
6690
6691 nvme_update_virt_res(n, sctrl, rt, nr);
6692 req->cqe.result = cpu_to_le32(nr);
6693
6694 return req->status;
6695 }
6696
6697 static uint16_t nvme_virt_set_state(NvmeCtrl *n, uint16_t cntlid, bool online)
6698 {
6699 PCIDevice *pci = PCI_DEVICE(n);
6700 NvmeCtrl *sn = NULL;
6701 NvmeSecCtrlEntry *sctrl;
6702 int vf_index;
6703
6704 sctrl = nvme_sctrl_for_cntlid(n, cntlid);
6705 if (!sctrl) {
6706 return NVME_INVALID_CTRL_ID | NVME_DNR;
6707 }
6708
6709 if (!pci_is_vf(pci)) {
6710 vf_index = le16_to_cpu(sctrl->vfn) - 1;
6711 sn = NVME(pcie_sriov_get_vf_at_index(pci, vf_index));
6712 }
6713
6714 if (online) {
6715 if (!sctrl->nvi || (le16_to_cpu(sctrl->nvq) < 2) || !sn) {
6716 return NVME_INVALID_SEC_CTRL_STATE | NVME_DNR;
6717 }
6718
6719 if (!sctrl->scs) {
6720 sctrl->scs = 0x1;
6721 nvme_ctrl_reset(sn, NVME_RESET_FUNCTION);
6722 }
6723 } else {
6724 nvme_update_virt_res(n, sctrl, NVME_VIRT_RES_INTERRUPT, 0);
6725 nvme_update_virt_res(n, sctrl, NVME_VIRT_RES_QUEUE, 0);
6726
6727 if (sctrl->scs) {
6728 sctrl->scs = 0x0;
6729 if (sn) {
6730 nvme_ctrl_reset(sn, NVME_RESET_FUNCTION);
6731 }
6732 }
6733 }
6734
6735 return NVME_SUCCESS;
6736 }
6737
6738 static uint16_t nvme_virt_mngmt(NvmeCtrl *n, NvmeRequest *req)
6739 {
6740 uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
6741 uint32_t dw11 = le32_to_cpu(req->cmd.cdw11);
6742 uint8_t act = dw10 & 0xf;
6743 uint8_t rt = (dw10 >> 8) & 0x7;
6744 uint16_t cntlid = (dw10 >> 16) & 0xffff;
6745 int nr = dw11 & 0xffff;
6746
6747 trace_pci_nvme_virt_mngmt(nvme_cid(req), act, cntlid, rt ? "VI" : "VQ", nr);
6748
6749 if (rt != NVME_VIRT_RES_QUEUE && rt != NVME_VIRT_RES_INTERRUPT) {
6750 return NVME_INVALID_RESOURCE_ID | NVME_DNR;
6751 }
6752
6753 switch (act) {
6754 case NVME_VIRT_MNGMT_ACTION_SEC_ASSIGN:
6755 return nvme_assign_virt_res_to_sec(n, req, cntlid, rt, nr);
6756 case NVME_VIRT_MNGMT_ACTION_PRM_ALLOC:
6757 return nvme_assign_virt_res_to_prim(n, req, cntlid, rt, nr);
6758 case NVME_VIRT_MNGMT_ACTION_SEC_ONLINE:
6759 return nvme_virt_set_state(n, cntlid, true);
6760 case NVME_VIRT_MNGMT_ACTION_SEC_OFFLINE:
6761 return nvme_virt_set_state(n, cntlid, false);
6762 default:
6763 return NVME_INVALID_FIELD | NVME_DNR;
6764 }
6765 }
6766
6767 static uint16_t nvme_dbbuf_config(NvmeCtrl *n, const NvmeRequest *req)
6768 {
6769 PCIDevice *pci = PCI_DEVICE(n);
6770 uint64_t dbs_addr = le64_to_cpu(req->cmd.dptr.prp1);
6771 uint64_t eis_addr = le64_to_cpu(req->cmd.dptr.prp2);
6772 int i;
6773
6774 /* Address should be page aligned */
6775 if (dbs_addr & (n->page_size - 1) || eis_addr & (n->page_size - 1)) {
6776 return NVME_INVALID_FIELD | NVME_DNR;
6777 }
6778
6779 /* Save shadow buffer base addr for use during queue creation */
6780 n->dbbuf_dbs = dbs_addr;
6781 n->dbbuf_eis = eis_addr;
6782 n->dbbuf_enabled = true;
6783
6784 for (i = 0; i < n->params.max_ioqpairs + 1; i++) {
6785 NvmeSQueue *sq = n->sq[i];
6786 NvmeCQueue *cq = n->cq[i];
6787
6788 if (sq) {
6789 /*
6790 * CAP.DSTRD is 0, so offset of ith sq db_addr is (i<<3)
6791 * nvme_process_db() uses this hard-coded way to calculate
6792 * doorbell offsets. Be consistent with that here.
6793 */
6794 sq->db_addr = dbs_addr + (i << 3);
6795 sq->ei_addr = eis_addr + (i << 3);
6796 pci_dma_write(pci, sq->db_addr, &sq->tail, sizeof(sq->tail));
6797
6798 if (n->params.ioeventfd && sq->sqid != 0) {
6799 if (!nvme_init_sq_ioeventfd(sq)) {
6800 sq->ioeventfd_enabled = true;
6801 }
6802 }
6803 }
6804
6805 if (cq) {
6806 /* CAP.DSTRD is 0, so offset of ith cq db_addr is (i<<3)+(1<<2) */
6807 cq->db_addr = dbs_addr + (i << 3) + (1 << 2);
6808 cq->ei_addr = eis_addr + (i << 3) + (1 << 2);
6809 pci_dma_write(pci, cq->db_addr, &cq->head, sizeof(cq->head));
6810
6811 if (n->params.ioeventfd && cq->cqid != 0) {
6812 if (!nvme_init_cq_ioeventfd(cq)) {
6813 cq->ioeventfd_enabled = true;
6814 }
6815 }
6816 }
6817 }
6818
6819 trace_pci_nvme_dbbuf_config(dbs_addr, eis_addr);
6820
6821 return NVME_SUCCESS;
6822 }
6823
6824 static uint16_t nvme_directive_send(NvmeCtrl *n, NvmeRequest *req)
6825 {
6826 return NVME_INVALID_FIELD | NVME_DNR;
6827 }
6828
6829 static uint16_t nvme_directive_receive(NvmeCtrl *n, NvmeRequest *req)
6830 {
6831 NvmeNamespace *ns;
6832 uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
6833 uint32_t dw11 = le32_to_cpu(req->cmd.cdw11);
6834 uint32_t nsid = le32_to_cpu(req->cmd.nsid);
6835 uint8_t doper, dtype;
6836 uint32_t numd, trans_len;
6837 NvmeDirectiveIdentify id = {
6838 .supported = 1 << NVME_DIRECTIVE_IDENTIFY,
6839 .enabled = 1 << NVME_DIRECTIVE_IDENTIFY,
6840 };
6841
6842 numd = dw10 + 1;
6843 doper = dw11 & 0xff;
6844 dtype = (dw11 >> 8) & 0xff;
6845
6846 trans_len = MIN(sizeof(NvmeDirectiveIdentify), numd << 2);
6847
6848 if (nsid == NVME_NSID_BROADCAST || dtype != NVME_DIRECTIVE_IDENTIFY ||
6849 doper != NVME_DIRECTIVE_RETURN_PARAMS) {
6850 return NVME_INVALID_FIELD | NVME_DNR;
6851 }
6852
6853 ns = nvme_ns(n, nsid);
6854 if (!ns) {
6855 return NVME_INVALID_FIELD | NVME_DNR;
6856 }
6857
6858 switch (dtype) {
6859 case NVME_DIRECTIVE_IDENTIFY:
6860 switch (doper) {
6861 case NVME_DIRECTIVE_RETURN_PARAMS:
6862 if (ns->endgrp->fdp.enabled) {
6863 id.supported |= 1 << NVME_DIRECTIVE_DATA_PLACEMENT;
6864 id.enabled |= 1 << NVME_DIRECTIVE_DATA_PLACEMENT;
6865 id.persistent |= 1 << NVME_DIRECTIVE_DATA_PLACEMENT;
6866 }
6867
6868 return nvme_c2h(n, (uint8_t *)&id, trans_len, req);
6869
6870 default:
6871 return NVME_INVALID_FIELD | NVME_DNR;
6872 }
6873
6874 default:
6875 return NVME_INVALID_FIELD;
6876 }
6877 }
6878
6879 static uint16_t nvme_admin_cmd(NvmeCtrl *n, NvmeRequest *req)
6880 {
6881 trace_pci_nvme_admin_cmd(nvme_cid(req), nvme_sqid(req), req->cmd.opcode,
6882 nvme_adm_opc_str(req->cmd.opcode));
6883
6884 if (!(nvme_cse_acs[req->cmd.opcode] & NVME_CMD_EFF_CSUPP)) {
6885 trace_pci_nvme_err_invalid_admin_opc(req->cmd.opcode);
6886 return NVME_INVALID_OPCODE | NVME_DNR;
6887 }
6888
6889 /* SGLs shall not be used for Admin commands in NVMe over PCIe */
6890 if (NVME_CMD_FLAGS_PSDT(req->cmd.flags) != NVME_PSDT_PRP) {
6891 return NVME_INVALID_FIELD | NVME_DNR;
6892 }
6893
6894 if (NVME_CMD_FLAGS_FUSE(req->cmd.flags)) {
6895 return NVME_INVALID_FIELD;
6896 }
6897
6898 switch (req->cmd.opcode) {
6899 case NVME_ADM_CMD_DELETE_SQ:
6900 return nvme_del_sq(n, req);
6901 case NVME_ADM_CMD_CREATE_SQ:
6902 return nvme_create_sq(n, req);
6903 case NVME_ADM_CMD_GET_LOG_PAGE:
6904 return nvme_get_log(n, req);
6905 case NVME_ADM_CMD_DELETE_CQ:
6906 return nvme_del_cq(n, req);
6907 case NVME_ADM_CMD_CREATE_CQ:
6908 return nvme_create_cq(n, req);
6909 case NVME_ADM_CMD_IDENTIFY:
6910 return nvme_identify(n, req);
6911 case NVME_ADM_CMD_ABORT:
6912 return nvme_abort(n, req);
6913 case NVME_ADM_CMD_SET_FEATURES:
6914 return nvme_set_feature(n, req);
6915 case NVME_ADM_CMD_GET_FEATURES:
6916 return nvme_get_feature(n, req);
6917 case NVME_ADM_CMD_ASYNC_EV_REQ:
6918 return nvme_aer(n, req);
6919 case NVME_ADM_CMD_NS_ATTACHMENT:
6920 return nvme_ns_attachment(n, req);
6921 case NVME_ADM_CMD_VIRT_MNGMT:
6922 return nvme_virt_mngmt(n, req);
6923 case NVME_ADM_CMD_DBBUF_CONFIG:
6924 return nvme_dbbuf_config(n, req);
6925 case NVME_ADM_CMD_FORMAT_NVM:
6926 return nvme_format(n, req);
6927 case NVME_ADM_CMD_DIRECTIVE_SEND:
6928 return nvme_directive_send(n, req);
6929 case NVME_ADM_CMD_DIRECTIVE_RECV:
6930 return nvme_directive_receive(n, req);
6931 default:
6932 assert(false);
6933 }
6934
6935 return NVME_INVALID_OPCODE | NVME_DNR;
6936 }
6937
6938 static void nvme_update_sq_eventidx(const NvmeSQueue *sq)
6939 {
6940 uint32_t v = cpu_to_le32(sq->tail);
6941
6942 trace_pci_nvme_update_sq_eventidx(sq->sqid, sq->tail);
6943
6944 pci_dma_write(PCI_DEVICE(sq->ctrl), sq->ei_addr, &v, sizeof(v));
6945 }
6946
6947 static void nvme_update_sq_tail(NvmeSQueue *sq)
6948 {
6949 uint32_t v;
6950
6951 pci_dma_read(PCI_DEVICE(sq->ctrl), sq->db_addr, &v, sizeof(v));
6952
6953 sq->tail = le32_to_cpu(v);
6954
6955 trace_pci_nvme_update_sq_tail(sq->sqid, sq->tail);
6956 }
6957
6958 static void nvme_process_sq(void *opaque)
6959 {
6960 NvmeSQueue *sq = opaque;
6961 NvmeCtrl *n = sq->ctrl;
6962 NvmeCQueue *cq = n->cq[sq->cqid];
6963
6964 uint16_t status;
6965 hwaddr addr;
6966 NvmeCmd cmd;
6967 NvmeRequest *req;
6968
6969 if (n->dbbuf_enabled) {
6970 nvme_update_sq_tail(sq);
6971 }
6972
6973 while (!(nvme_sq_empty(sq) || QTAILQ_EMPTY(&sq->req_list))) {
6974 addr = sq->dma_addr + sq->head * n->sqe_size;
6975 if (nvme_addr_read(n, addr, (void *)&cmd, sizeof(cmd))) {
6976 trace_pci_nvme_err_addr_read(addr);
6977 trace_pci_nvme_err_cfs();
6978 stl_le_p(&n->bar.csts, NVME_CSTS_FAILED);
6979 break;
6980 }
6981 nvme_inc_sq_head(sq);
6982
6983 req = QTAILQ_FIRST(&sq->req_list);
6984 QTAILQ_REMOVE(&sq->req_list, req, entry);
6985 QTAILQ_INSERT_TAIL(&sq->out_req_list, req, entry);
6986 nvme_req_clear(req);
6987 req->cqe.cid = cmd.cid;
6988 memcpy(&req->cmd, &cmd, sizeof(NvmeCmd));
6989
6990 status = sq->sqid ? nvme_io_cmd(n, req) :
6991 nvme_admin_cmd(n, req);
6992 if (status != NVME_NO_COMPLETE) {
6993 req->status = status;
6994 nvme_enqueue_req_completion(cq, req);
6995 }
6996
6997 if (n->dbbuf_enabled) {
6998 nvme_update_sq_eventidx(sq);
6999 nvme_update_sq_tail(sq);
7000 }
7001 }
7002 }
7003
7004 static void nvme_update_msixcap_ts(PCIDevice *pci_dev, uint32_t table_size)
7005 {
7006 uint8_t *config;
7007
7008 if (!msix_present(pci_dev)) {
7009 return;
7010 }
7011
7012 assert(table_size > 0 && table_size <= pci_dev->msix_entries_nr);
7013
7014 config = pci_dev->config + pci_dev->msix_cap;
7015 pci_set_word_by_mask(config + PCI_MSIX_FLAGS, PCI_MSIX_FLAGS_QSIZE,
7016 table_size - 1);
7017 }
7018
7019 static void nvme_activate_virt_res(NvmeCtrl *n)
7020 {
7021 PCIDevice *pci_dev = PCI_DEVICE(n);
7022 NvmePriCtrlCap *cap = &n->pri_ctrl_cap;
7023 NvmeSecCtrlEntry *sctrl;
7024
7025 /* -1 to account for the admin queue */
7026 if (pci_is_vf(pci_dev)) {
7027 sctrl = nvme_sctrl(n);
7028 cap->vqprt = sctrl->nvq;
7029 cap->viprt = sctrl->nvi;
7030 n->conf_ioqpairs = sctrl->nvq ? le16_to_cpu(sctrl->nvq) - 1 : 0;
7031 n->conf_msix_qsize = sctrl->nvi ? le16_to_cpu(sctrl->nvi) : 1;
7032 } else {
7033 cap->vqrfap = n->next_pri_ctrl_cap.vqrfap;
7034 cap->virfap = n->next_pri_ctrl_cap.virfap;
7035 n->conf_ioqpairs = le16_to_cpu(cap->vqprt) +
7036 le16_to_cpu(cap->vqrfap) - 1;
7037 n->conf_msix_qsize = le16_to_cpu(cap->viprt) +
7038 le16_to_cpu(cap->virfap);
7039 }
7040 }
7041
7042 static void nvme_ctrl_reset(NvmeCtrl *n, NvmeResetType rst)
7043 {
7044 PCIDevice *pci_dev = PCI_DEVICE(n);
7045 NvmeSecCtrlEntry *sctrl;
7046 NvmeNamespace *ns;
7047 int i;
7048
7049 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
7050 ns = nvme_ns(n, i);
7051 if (!ns) {
7052 continue;
7053 }
7054
7055 nvme_ns_drain(ns);
7056 }
7057
7058 for (i = 0; i < n->params.max_ioqpairs + 1; i++) {
7059 if (n->sq[i] != NULL) {
7060 nvme_free_sq(n->sq[i], n);
7061 }
7062 }
7063 for (i = 0; i < n->params.max_ioqpairs + 1; i++) {
7064 if (n->cq[i] != NULL) {
7065 nvme_free_cq(n->cq[i], n);
7066 }
7067 }
7068
7069 while (!QTAILQ_EMPTY(&n->aer_queue)) {
7070 NvmeAsyncEvent *event = QTAILQ_FIRST(&n->aer_queue);
7071 QTAILQ_REMOVE(&n->aer_queue, event, entry);
7072 g_free(event);
7073 }
7074
7075 if (n->params.sriov_max_vfs) {
7076 if (!pci_is_vf(pci_dev)) {
7077 for (i = 0; i < n->sec_ctrl_list.numcntl; i++) {
7078 sctrl = &n->sec_ctrl_list.sec[i];
7079 nvme_virt_set_state(n, le16_to_cpu(sctrl->scid), false);
7080 }
7081
7082 if (rst != NVME_RESET_CONTROLLER) {
7083 pcie_sriov_pf_disable_vfs(pci_dev);
7084 }
7085 }
7086
7087 if (rst != NVME_RESET_CONTROLLER) {
7088 nvme_activate_virt_res(n);
7089 }
7090 }
7091
7092 n->aer_queued = 0;
7093 n->aer_mask = 0;
7094 n->outstanding_aers = 0;
7095 n->qs_created = false;
7096
7097 nvme_update_msixcap_ts(pci_dev, n->conf_msix_qsize);
7098
7099 if (pci_is_vf(pci_dev)) {
7100 sctrl = nvme_sctrl(n);
7101
7102 stl_le_p(&n->bar.csts, sctrl->scs ? 0 : NVME_CSTS_FAILED);
7103 } else {
7104 stl_le_p(&n->bar.csts, 0);
7105 }
7106
7107 stl_le_p(&n->bar.intms, 0);
7108 stl_le_p(&n->bar.intmc, 0);
7109 stl_le_p(&n->bar.cc, 0);
7110
7111 n->dbbuf_dbs = 0;
7112 n->dbbuf_eis = 0;
7113 n->dbbuf_enabled = false;
7114 }
7115
7116 static void nvme_ctrl_shutdown(NvmeCtrl *n)
7117 {
7118 NvmeNamespace *ns;
7119 int i;
7120
7121 if (n->pmr.dev) {
7122 memory_region_msync(&n->pmr.dev->mr, 0, n->pmr.dev->size);
7123 }
7124
7125 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
7126 ns = nvme_ns(n, i);
7127 if (!ns) {
7128 continue;
7129 }
7130
7131 nvme_ns_shutdown(ns);
7132 }
7133 }
7134
7135 static void nvme_select_iocs(NvmeCtrl *n)
7136 {
7137 NvmeNamespace *ns;
7138 int i;
7139
7140 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
7141 ns = nvme_ns(n, i);
7142 if (!ns) {
7143 continue;
7144 }
7145
7146 nvme_select_iocs_ns(n, ns);
7147 }
7148 }
7149
7150 static int nvme_start_ctrl(NvmeCtrl *n)
7151 {
7152 uint64_t cap = ldq_le_p(&n->bar.cap);
7153 uint32_t cc = ldl_le_p(&n->bar.cc);
7154 uint32_t aqa = ldl_le_p(&n->bar.aqa);
7155 uint64_t asq = ldq_le_p(&n->bar.asq);
7156 uint64_t acq = ldq_le_p(&n->bar.acq);
7157 uint32_t page_bits = NVME_CC_MPS(cc) + 12;
7158 uint32_t page_size = 1 << page_bits;
7159 NvmeSecCtrlEntry *sctrl = nvme_sctrl(n);
7160
7161 if (pci_is_vf(PCI_DEVICE(n)) && !sctrl->scs) {
7162 trace_pci_nvme_err_startfail_virt_state(le16_to_cpu(sctrl->nvi),
7163 le16_to_cpu(sctrl->nvq));
7164 return -1;
7165 }
7166 if (unlikely(n->cq[0])) {
7167 trace_pci_nvme_err_startfail_cq();
7168 return -1;
7169 }
7170 if (unlikely(n->sq[0])) {
7171 trace_pci_nvme_err_startfail_sq();
7172 return -1;
7173 }
7174 if (unlikely(asq & (page_size - 1))) {
7175 trace_pci_nvme_err_startfail_asq_misaligned(asq);
7176 return -1;
7177 }
7178 if (unlikely(acq & (page_size - 1))) {
7179 trace_pci_nvme_err_startfail_acq_misaligned(acq);
7180 return -1;
7181 }
7182 if (unlikely(!(NVME_CAP_CSS(cap) & (1 << NVME_CC_CSS(cc))))) {
7183 trace_pci_nvme_err_startfail_css(NVME_CC_CSS(cc));
7184 return -1;
7185 }
7186 if (unlikely(NVME_CC_MPS(cc) < NVME_CAP_MPSMIN(cap))) {
7187 trace_pci_nvme_err_startfail_page_too_small(
7188 NVME_CC_MPS(cc),
7189 NVME_CAP_MPSMIN(cap));
7190 return -1;
7191 }
7192 if (unlikely(NVME_CC_MPS(cc) >
7193 NVME_CAP_MPSMAX(cap))) {
7194 trace_pci_nvme_err_startfail_page_too_large(
7195 NVME_CC_MPS(cc),
7196 NVME_CAP_MPSMAX(cap));
7197 return -1;
7198 }
7199 if (unlikely(NVME_CC_IOCQES(cc) <
7200 NVME_CTRL_CQES_MIN(n->id_ctrl.cqes))) {
7201 trace_pci_nvme_err_startfail_cqent_too_small(
7202 NVME_CC_IOCQES(cc),
7203 NVME_CTRL_CQES_MIN(cap));
7204 return -1;
7205 }
7206 if (unlikely(NVME_CC_IOCQES(cc) >
7207 NVME_CTRL_CQES_MAX(n->id_ctrl.cqes))) {
7208 trace_pci_nvme_err_startfail_cqent_too_large(
7209 NVME_CC_IOCQES(cc),
7210 NVME_CTRL_CQES_MAX(cap));
7211 return -1;
7212 }
7213 if (unlikely(NVME_CC_IOSQES(cc) <
7214 NVME_CTRL_SQES_MIN(n->id_ctrl.sqes))) {
7215 trace_pci_nvme_err_startfail_sqent_too_small(
7216 NVME_CC_IOSQES(cc),
7217 NVME_CTRL_SQES_MIN(cap));
7218 return -1;
7219 }
7220 if (unlikely(NVME_CC_IOSQES(cc) >
7221 NVME_CTRL_SQES_MAX(n->id_ctrl.sqes))) {
7222 trace_pci_nvme_err_startfail_sqent_too_large(
7223 NVME_CC_IOSQES(cc),
7224 NVME_CTRL_SQES_MAX(cap));
7225 return -1;
7226 }
7227 if (unlikely(!NVME_AQA_ASQS(aqa))) {
7228 trace_pci_nvme_err_startfail_asqent_sz_zero();
7229 return -1;
7230 }
7231 if (unlikely(!NVME_AQA_ACQS(aqa))) {
7232 trace_pci_nvme_err_startfail_acqent_sz_zero();
7233 return -1;
7234 }
7235
7236 n->page_bits = page_bits;
7237 n->page_size = page_size;
7238 n->max_prp_ents = n->page_size / sizeof(uint64_t);
7239 n->cqe_size = 1 << NVME_CC_IOCQES(cc);
7240 n->sqe_size = 1 << NVME_CC_IOSQES(cc);
7241 nvme_init_cq(&n->admin_cq, n, acq, 0, 0, NVME_AQA_ACQS(aqa) + 1, 1);
7242 nvme_init_sq(&n->admin_sq, n, asq, 0, 0, NVME_AQA_ASQS(aqa) + 1);
7243
7244 nvme_set_timestamp(n, 0ULL);
7245
7246 nvme_select_iocs(n);
7247
7248 return 0;
7249 }
7250
7251 static void nvme_cmb_enable_regs(NvmeCtrl *n)
7252 {
7253 uint32_t cmbloc = ldl_le_p(&n->bar.cmbloc);
7254 uint32_t cmbsz = ldl_le_p(&n->bar.cmbsz);
7255
7256 NVME_CMBLOC_SET_CDPCILS(cmbloc, 1);
7257 NVME_CMBLOC_SET_CDPMLS(cmbloc, 1);
7258 NVME_CMBLOC_SET_BIR(cmbloc, NVME_CMB_BIR);
7259 stl_le_p(&n->bar.cmbloc, cmbloc);
7260
7261 NVME_CMBSZ_SET_SQS(cmbsz, 1);
7262 NVME_CMBSZ_SET_CQS(cmbsz, 0);
7263 NVME_CMBSZ_SET_LISTS(cmbsz, 1);
7264 NVME_CMBSZ_SET_RDS(cmbsz, 1);
7265 NVME_CMBSZ_SET_WDS(cmbsz, 1);
7266 NVME_CMBSZ_SET_SZU(cmbsz, 2); /* MBs */
7267 NVME_CMBSZ_SET_SZ(cmbsz, n->params.cmb_size_mb);
7268 stl_le_p(&n->bar.cmbsz, cmbsz);
7269 }
7270
7271 static void nvme_write_bar(NvmeCtrl *n, hwaddr offset, uint64_t data,
7272 unsigned size)
7273 {
7274 PCIDevice *pci = PCI_DEVICE(n);
7275 uint64_t cap = ldq_le_p(&n->bar.cap);
7276 uint32_t cc = ldl_le_p(&n->bar.cc);
7277 uint32_t intms = ldl_le_p(&n->bar.intms);
7278 uint32_t csts = ldl_le_p(&n->bar.csts);
7279 uint32_t pmrsts = ldl_le_p(&n->bar.pmrsts);
7280
7281 if (unlikely(offset & (sizeof(uint32_t) - 1))) {
7282 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_misaligned32,
7283 "MMIO write not 32-bit aligned,"
7284 " offset=0x%"PRIx64"", offset);
7285 /* should be ignored, fall through for now */
7286 }
7287
7288 if (unlikely(size < sizeof(uint32_t))) {
7289 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_toosmall,
7290 "MMIO write smaller than 32-bits,"
7291 " offset=0x%"PRIx64", size=%u",
7292 offset, size);
7293 /* should be ignored, fall through for now */
7294 }
7295
7296 switch (offset) {
7297 case NVME_REG_INTMS:
7298 if (unlikely(msix_enabled(pci))) {
7299 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix,
7300 "undefined access to interrupt mask set"
7301 " when MSI-X is enabled");
7302 /* should be ignored, fall through for now */
7303 }
7304 intms |= data;
7305 stl_le_p(&n->bar.intms, intms);
7306 n->bar.intmc = n->bar.intms;
7307 trace_pci_nvme_mmio_intm_set(data & 0xffffffff, intms);
7308 nvme_irq_check(n);
7309 break;
7310 case NVME_REG_INTMC:
7311 if (unlikely(msix_enabled(pci))) {
7312 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix,
7313 "undefined access to interrupt mask clr"
7314 " when MSI-X is enabled");
7315 /* should be ignored, fall through for now */
7316 }
7317 intms &= ~data;
7318 stl_le_p(&n->bar.intms, intms);
7319 n->bar.intmc = n->bar.intms;
7320 trace_pci_nvme_mmio_intm_clr(data & 0xffffffff, intms);
7321 nvme_irq_check(n);
7322 break;
7323 case NVME_REG_CC:
7324 stl_le_p(&n->bar.cc, data);
7325
7326 trace_pci_nvme_mmio_cfg(data & 0xffffffff);
7327
7328 if (NVME_CC_SHN(data) && !(NVME_CC_SHN(cc))) {
7329 trace_pci_nvme_mmio_shutdown_set();
7330 nvme_ctrl_shutdown(n);
7331 csts &= ~(CSTS_SHST_MASK << CSTS_SHST_SHIFT);
7332 csts |= NVME_CSTS_SHST_COMPLETE;
7333 } else if (!NVME_CC_SHN(data) && NVME_CC_SHN(cc)) {
7334 trace_pci_nvme_mmio_shutdown_cleared();
7335 csts &= ~(CSTS_SHST_MASK << CSTS_SHST_SHIFT);
7336 }
7337
7338 if (NVME_CC_EN(data) && !NVME_CC_EN(cc)) {
7339 if (unlikely(nvme_start_ctrl(n))) {
7340 trace_pci_nvme_err_startfail();
7341 csts = NVME_CSTS_FAILED;
7342 } else {
7343 trace_pci_nvme_mmio_start_success();
7344 csts = NVME_CSTS_READY;
7345 }
7346 } else if (!NVME_CC_EN(data) && NVME_CC_EN(cc)) {
7347 trace_pci_nvme_mmio_stopped();
7348 nvme_ctrl_reset(n, NVME_RESET_CONTROLLER);
7349
7350 break;
7351 }
7352
7353 stl_le_p(&n->bar.csts, csts);
7354
7355 break;
7356 case NVME_REG_CSTS:
7357 if (data & (1 << 4)) {
7358 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ssreset_w1c_unsupported,
7359 "attempted to W1C CSTS.NSSRO"
7360 " but CAP.NSSRS is zero (not supported)");
7361 } else if (data != 0) {
7362 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ro_csts,
7363 "attempted to set a read only bit"
7364 " of controller status");
7365 }
7366 break;
7367 case NVME_REG_NSSR:
7368 if (data == 0x4e564d65) {
7369 trace_pci_nvme_ub_mmiowr_ssreset_unsupported();
7370 } else {
7371 /* The spec says that writes of other values have no effect */
7372 return;
7373 }
7374 break;
7375 case NVME_REG_AQA:
7376 stl_le_p(&n->bar.aqa, data);
7377 trace_pci_nvme_mmio_aqattr(data & 0xffffffff);
7378 break;
7379 case NVME_REG_ASQ:
7380 stn_le_p(&n->bar.asq, size, data);
7381 trace_pci_nvme_mmio_asqaddr(data);
7382 break;
7383 case NVME_REG_ASQ + 4:
7384 stl_le_p((uint8_t *)&n->bar.asq + 4, data);
7385 trace_pci_nvme_mmio_asqaddr_hi(data, ldq_le_p(&n->bar.asq));
7386 break;
7387 case NVME_REG_ACQ:
7388 trace_pci_nvme_mmio_acqaddr(data);
7389 stn_le_p(&n->bar.acq, size, data);
7390 break;
7391 case NVME_REG_ACQ + 4:
7392 stl_le_p((uint8_t *)&n->bar.acq + 4, data);
7393 trace_pci_nvme_mmio_acqaddr_hi(data, ldq_le_p(&n->bar.acq));
7394 break;
7395 case NVME_REG_CMBLOC:
7396 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbloc_reserved,
7397 "invalid write to reserved CMBLOC"
7398 " when CMBSZ is zero, ignored");
7399 return;
7400 case NVME_REG_CMBSZ:
7401 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbsz_readonly,
7402 "invalid write to read only CMBSZ, ignored");
7403 return;
7404 case NVME_REG_CMBMSC:
7405 if (!NVME_CAP_CMBS(cap)) {
7406 return;
7407 }
7408
7409 stn_le_p(&n->bar.cmbmsc, size, data);
7410 n->cmb.cmse = false;
7411
7412 if (NVME_CMBMSC_CRE(data)) {
7413 nvme_cmb_enable_regs(n);
7414
7415 if (NVME_CMBMSC_CMSE(data)) {
7416 uint64_t cmbmsc = ldq_le_p(&n->bar.cmbmsc);
7417 hwaddr cba = NVME_CMBMSC_CBA(cmbmsc) << CMBMSC_CBA_SHIFT;
7418 if (cba + int128_get64(n->cmb.mem.size) < cba) {
7419 uint32_t cmbsts = ldl_le_p(&n->bar.cmbsts);
7420 NVME_CMBSTS_SET_CBAI(cmbsts, 1);
7421 stl_le_p(&n->bar.cmbsts, cmbsts);
7422 return;
7423 }
7424
7425 n->cmb.cba = cba;
7426 n->cmb.cmse = true;
7427 }
7428 } else {
7429 n->bar.cmbsz = 0;
7430 n->bar.cmbloc = 0;
7431 }
7432
7433 return;
7434 case NVME_REG_CMBMSC + 4:
7435 stl_le_p((uint8_t *)&n->bar.cmbmsc + 4, data);
7436 return;
7437
7438 case NVME_REG_PMRCAP:
7439 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrcap_readonly,
7440 "invalid write to PMRCAP register, ignored");
7441 return;
7442 case NVME_REG_PMRCTL:
7443 if (!NVME_CAP_PMRS(cap)) {
7444 return;
7445 }
7446
7447 stl_le_p(&n->bar.pmrctl, data);
7448 if (NVME_PMRCTL_EN(data)) {
7449 memory_region_set_enabled(&n->pmr.dev->mr, true);
7450 pmrsts = 0;
7451 } else {
7452 memory_region_set_enabled(&n->pmr.dev->mr, false);
7453 NVME_PMRSTS_SET_NRDY(pmrsts, 1);
7454 n->pmr.cmse = false;
7455 }
7456 stl_le_p(&n->bar.pmrsts, pmrsts);
7457 return;
7458 case NVME_REG_PMRSTS:
7459 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrsts_readonly,
7460 "invalid write to PMRSTS register, ignored");
7461 return;
7462 case NVME_REG_PMREBS:
7463 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrebs_readonly,
7464 "invalid write to PMREBS register, ignored");
7465 return;
7466 case NVME_REG_PMRSWTP:
7467 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrswtp_readonly,
7468 "invalid write to PMRSWTP register, ignored");
7469 return;
7470 case NVME_REG_PMRMSCL:
7471 if (!NVME_CAP_PMRS(cap)) {
7472 return;
7473 }
7474
7475 stl_le_p(&n->bar.pmrmscl, data);
7476 n->pmr.cmse = false;
7477
7478 if (NVME_PMRMSCL_CMSE(data)) {
7479 uint64_t pmrmscu = ldl_le_p(&n->bar.pmrmscu);
7480 hwaddr cba = pmrmscu << 32 |
7481 (NVME_PMRMSCL_CBA(data) << PMRMSCL_CBA_SHIFT);
7482 if (cba + int128_get64(n->pmr.dev->mr.size) < cba) {
7483 NVME_PMRSTS_SET_CBAI(pmrsts, 1);
7484 stl_le_p(&n->bar.pmrsts, pmrsts);
7485 return;
7486 }
7487
7488 n->pmr.cmse = true;
7489 n->pmr.cba = cba;
7490 }
7491
7492 return;
7493 case NVME_REG_PMRMSCU:
7494 if (!NVME_CAP_PMRS(cap)) {
7495 return;
7496 }
7497
7498 stl_le_p(&n->bar.pmrmscu, data);
7499 return;
7500 default:
7501 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_invalid,
7502 "invalid MMIO write,"
7503 " offset=0x%"PRIx64", data=%"PRIx64"",
7504 offset, data);
7505 break;
7506 }
7507 }
7508
7509 static uint64_t nvme_mmio_read(void *opaque, hwaddr addr, unsigned size)
7510 {
7511 NvmeCtrl *n = (NvmeCtrl *)opaque;
7512 uint8_t *ptr = (uint8_t *)&n->bar;
7513
7514 trace_pci_nvme_mmio_read(addr, size);
7515
7516 if (unlikely(addr & (sizeof(uint32_t) - 1))) {
7517 NVME_GUEST_ERR(pci_nvme_ub_mmiord_misaligned32,
7518 "MMIO read not 32-bit aligned,"
7519 " offset=0x%"PRIx64"", addr);
7520 /* should RAZ, fall through for now */
7521 } else if (unlikely(size < sizeof(uint32_t))) {
7522 NVME_GUEST_ERR(pci_nvme_ub_mmiord_toosmall,
7523 "MMIO read smaller than 32-bits,"
7524 " offset=0x%"PRIx64"", addr);
7525 /* should RAZ, fall through for now */
7526 }
7527
7528 if (addr > sizeof(n->bar) - size) {
7529 NVME_GUEST_ERR(pci_nvme_ub_mmiord_invalid_ofs,
7530 "MMIO read beyond last register,"
7531 " offset=0x%"PRIx64", returning 0", addr);
7532
7533 return 0;
7534 }
7535
7536 if (pci_is_vf(PCI_DEVICE(n)) && !nvme_sctrl(n)->scs &&
7537 addr != NVME_REG_CSTS) {
7538 trace_pci_nvme_err_ignored_mmio_vf_offline(addr, size);
7539 return 0;
7540 }
7541
7542 /*
7543 * When PMRWBM bit 1 is set then read from
7544 * from PMRSTS should ensure prior writes
7545 * made it to persistent media
7546 */
7547 if (addr == NVME_REG_PMRSTS &&
7548 (NVME_PMRCAP_PMRWBM(ldl_le_p(&n->bar.pmrcap)) & 0x02)) {
7549 memory_region_msync(&n->pmr.dev->mr, 0, n->pmr.dev->size);
7550 }
7551
7552 return ldn_le_p(ptr + addr, size);
7553 }
7554
7555 static void nvme_process_db(NvmeCtrl *n, hwaddr addr, int val)
7556 {
7557 PCIDevice *pci = PCI_DEVICE(n);
7558 uint32_t qid;
7559
7560 if (unlikely(addr & ((1 << 2) - 1))) {
7561 NVME_GUEST_ERR(pci_nvme_ub_db_wr_misaligned,
7562 "doorbell write not 32-bit aligned,"
7563 " offset=0x%"PRIx64", ignoring", addr);
7564 return;
7565 }
7566
7567 if (((addr - 0x1000) >> 2) & 1) {
7568 /* Completion queue doorbell write */
7569
7570 uint16_t new_head = val & 0xffff;
7571 int start_sqs;
7572 NvmeCQueue *cq;
7573
7574 qid = (addr - (0x1000 + (1 << 2))) >> 3;
7575 if (unlikely(nvme_check_cqid(n, qid))) {
7576 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cq,
7577 "completion queue doorbell write"
7578 " for nonexistent queue,"
7579 " sqid=%"PRIu32", ignoring", qid);
7580
7581 /*
7582 * NVM Express v1.3d, Section 4.1 state: "If host software writes
7583 * an invalid value to the Submission Queue Tail Doorbell or
7584 * Completion Queue Head Doorbell regiter and an Asynchronous Event
7585 * Request command is outstanding, then an asynchronous event is
7586 * posted to the Admin Completion Queue with a status code of
7587 * Invalid Doorbell Write Value."
7588 *
7589 * Also note that the spec includes the "Invalid Doorbell Register"
7590 * status code, but nowhere does it specify when to use it.
7591 * However, it seems reasonable to use it here in a similar
7592 * fashion.
7593 */
7594 if (n->outstanding_aers) {
7595 nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
7596 NVME_AER_INFO_ERR_INVALID_DB_REGISTER,
7597 NVME_LOG_ERROR_INFO);
7598 }
7599
7600 return;
7601 }
7602
7603 cq = n->cq[qid];
7604 if (unlikely(new_head >= cq->size)) {
7605 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cqhead,
7606 "completion queue doorbell write value"
7607 " beyond queue size, sqid=%"PRIu32","
7608 " new_head=%"PRIu16", ignoring",
7609 qid, new_head);
7610
7611 if (n->outstanding_aers) {
7612 nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
7613 NVME_AER_INFO_ERR_INVALID_DB_VALUE,
7614 NVME_LOG_ERROR_INFO);
7615 }
7616
7617 return;
7618 }
7619
7620 trace_pci_nvme_mmio_doorbell_cq(cq->cqid, new_head);
7621
7622 start_sqs = nvme_cq_full(cq) ? 1 : 0;
7623 cq->head = new_head;
7624 if (!qid && n->dbbuf_enabled) {
7625 pci_dma_write(pci, cq->db_addr, &cq->head, sizeof(cq->head));
7626 }
7627 if (start_sqs) {
7628 NvmeSQueue *sq;
7629 QTAILQ_FOREACH(sq, &cq->sq_list, entry) {
7630 qemu_bh_schedule(sq->bh);
7631 }
7632 qemu_bh_schedule(cq->bh);
7633 }
7634
7635 if (cq->tail == cq->head) {
7636 if (cq->irq_enabled) {
7637 n->cq_pending--;
7638 }
7639
7640 nvme_irq_deassert(n, cq);
7641 }
7642 } else {
7643 /* Submission queue doorbell write */
7644
7645 uint16_t new_tail = val & 0xffff;
7646 NvmeSQueue *sq;
7647
7648 qid = (addr - 0x1000) >> 3;
7649 if (unlikely(nvme_check_sqid(n, qid))) {
7650 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sq,
7651 "submission queue doorbell write"
7652 " for nonexistent queue,"
7653 " sqid=%"PRIu32", ignoring", qid);
7654
7655 if (n->outstanding_aers) {
7656 nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
7657 NVME_AER_INFO_ERR_INVALID_DB_REGISTER,
7658 NVME_LOG_ERROR_INFO);
7659 }
7660
7661 return;
7662 }
7663
7664 sq = n->sq[qid];
7665 if (unlikely(new_tail >= sq->size)) {
7666 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sqtail,
7667 "submission queue doorbell write value"
7668 " beyond queue size, sqid=%"PRIu32","
7669 " new_tail=%"PRIu16", ignoring",
7670 qid, new_tail);
7671
7672 if (n->outstanding_aers) {
7673 nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
7674 NVME_AER_INFO_ERR_INVALID_DB_VALUE,
7675 NVME_LOG_ERROR_INFO);
7676 }
7677
7678 return;
7679 }
7680
7681 trace_pci_nvme_mmio_doorbell_sq(sq->sqid, new_tail);
7682
7683 sq->tail = new_tail;
7684 if (!qid && n->dbbuf_enabled) {
7685 /*
7686 * The spec states "the host shall also update the controller's
7687 * corresponding doorbell property to match the value of that entry
7688 * in the Shadow Doorbell buffer."
7689 *
7690 * Since this context is currently a VM trap, we can safely enforce
7691 * the requirement from the device side in case the host is
7692 * misbehaving.
7693 *
7694 * Note, we shouldn't have to do this, but various drivers
7695 * including ones that run on Linux, are not updating Admin Queues,
7696 * so we can't trust reading it for an appropriate sq tail.
7697 */
7698 pci_dma_write(pci, sq->db_addr, &sq->tail, sizeof(sq->tail));
7699 }
7700
7701 qemu_bh_schedule(sq->bh);
7702 }
7703 }
7704
7705 static void nvme_mmio_write(void *opaque, hwaddr addr, uint64_t data,
7706 unsigned size)
7707 {
7708 NvmeCtrl *n = (NvmeCtrl *)opaque;
7709
7710 trace_pci_nvme_mmio_write(addr, data, size);
7711
7712 if (pci_is_vf(PCI_DEVICE(n)) && !nvme_sctrl(n)->scs &&
7713 addr != NVME_REG_CSTS) {
7714 trace_pci_nvme_err_ignored_mmio_vf_offline(addr, size);
7715 return;
7716 }
7717
7718 if (addr < sizeof(n->bar)) {
7719 nvme_write_bar(n, addr, data, size);
7720 } else {
7721 nvme_process_db(n, addr, data);
7722 }
7723 }
7724
7725 static const MemoryRegionOps nvme_mmio_ops = {
7726 .read = nvme_mmio_read,
7727 .write = nvme_mmio_write,
7728 .endianness = DEVICE_LITTLE_ENDIAN,
7729 .impl = {
7730 .min_access_size = 2,
7731 .max_access_size = 8,
7732 },
7733 };
7734
7735 static void nvme_cmb_write(void *opaque, hwaddr addr, uint64_t data,
7736 unsigned size)
7737 {
7738 NvmeCtrl *n = (NvmeCtrl *)opaque;
7739 stn_le_p(&n->cmb.buf[addr], size, data);
7740 }
7741
7742 static uint64_t nvme_cmb_read(void *opaque, hwaddr addr, unsigned size)
7743 {
7744 NvmeCtrl *n = (NvmeCtrl *)opaque;
7745 return ldn_le_p(&n->cmb.buf[addr], size);
7746 }
7747
7748 static const MemoryRegionOps nvme_cmb_ops = {
7749 .read = nvme_cmb_read,
7750 .write = nvme_cmb_write,
7751 .endianness = DEVICE_LITTLE_ENDIAN,
7752 .impl = {
7753 .min_access_size = 1,
7754 .max_access_size = 8,
7755 },
7756 };
7757
7758 static bool nvme_check_params(NvmeCtrl *n, Error **errp)
7759 {
7760 NvmeParams *params = &n->params;
7761
7762 if (params->num_queues) {
7763 warn_report("num_queues is deprecated; please use max_ioqpairs "
7764 "instead");
7765
7766 params->max_ioqpairs = params->num_queues - 1;
7767 }
7768
7769 if (n->namespace.blkconf.blk && n->subsys) {
7770 error_setg(errp, "subsystem support is unavailable with legacy "
7771 "namespace ('drive' property)");
7772 return false;
7773 }
7774
7775 if (params->max_ioqpairs < 1 ||
7776 params->max_ioqpairs > NVME_MAX_IOQPAIRS) {
7777 error_setg(errp, "max_ioqpairs must be between 1 and %d",
7778 NVME_MAX_IOQPAIRS);
7779 return false;
7780 }
7781
7782 if (params->msix_qsize < 1 ||
7783 params->msix_qsize > PCI_MSIX_FLAGS_QSIZE + 1) {
7784 error_setg(errp, "msix_qsize must be between 1 and %d",
7785 PCI_MSIX_FLAGS_QSIZE + 1);
7786 return false;
7787 }
7788
7789 if (!params->serial) {
7790 error_setg(errp, "serial property not set");
7791 return false;
7792 }
7793
7794 if (n->pmr.dev) {
7795 if (host_memory_backend_is_mapped(n->pmr.dev)) {
7796 error_setg(errp, "can't use already busy memdev: %s",
7797 object_get_canonical_path_component(OBJECT(n->pmr.dev)));
7798 return false;
7799 }
7800
7801 if (!is_power_of_2(n->pmr.dev->size)) {
7802 error_setg(errp, "pmr backend size needs to be power of 2 in size");
7803 return false;
7804 }
7805
7806 host_memory_backend_set_mapped(n->pmr.dev, true);
7807 }
7808
7809 if (n->params.zasl > n->params.mdts) {
7810 error_setg(errp, "zoned.zasl (Zone Append Size Limit) must be less "
7811 "than or equal to mdts (Maximum Data Transfer Size)");
7812 return false;
7813 }
7814
7815 if (!n->params.vsl) {
7816 error_setg(errp, "vsl must be non-zero");
7817 return false;
7818 }
7819
7820 if (params->sriov_max_vfs) {
7821 if (!n->subsys) {
7822 error_setg(errp, "subsystem is required for the use of SR-IOV");
7823 return false;
7824 }
7825
7826 if (params->sriov_max_vfs > NVME_MAX_VFS) {
7827 error_setg(errp, "sriov_max_vfs must be between 0 and %d",
7828 NVME_MAX_VFS);
7829 return false;
7830 }
7831
7832 if (params->cmb_size_mb) {
7833 error_setg(errp, "CMB is not supported with SR-IOV");
7834 return false;
7835 }
7836
7837 if (n->pmr.dev) {
7838 error_setg(errp, "PMR is not supported with SR-IOV");
7839 return false;
7840 }
7841
7842 if (!params->sriov_vq_flexible || !params->sriov_vi_flexible) {
7843 error_setg(errp, "both sriov_vq_flexible and sriov_vi_flexible"
7844 " must be set for the use of SR-IOV");
7845 return false;
7846 }
7847
7848 if (params->sriov_vq_flexible < params->sriov_max_vfs * 2) {
7849 error_setg(errp, "sriov_vq_flexible must be greater than or equal"
7850 " to %d (sriov_max_vfs * 2)", params->sriov_max_vfs * 2);
7851 return false;
7852 }
7853
7854 if (params->max_ioqpairs < params->sriov_vq_flexible + 2) {
7855 error_setg(errp, "(max_ioqpairs - sriov_vq_flexible) must be"
7856 " greater than or equal to 2");
7857 return false;
7858 }
7859
7860 if (params->sriov_vi_flexible < params->sriov_max_vfs) {
7861 error_setg(errp, "sriov_vi_flexible must be greater than or equal"
7862 " to %d (sriov_max_vfs)", params->sriov_max_vfs);
7863 return false;
7864 }
7865
7866 if (params->msix_qsize < params->sriov_vi_flexible + 1) {
7867 error_setg(errp, "(msix_qsize - sriov_vi_flexible) must be"
7868 " greater than or equal to 1");
7869 return false;
7870 }
7871
7872 if (params->sriov_max_vi_per_vf &&
7873 (params->sriov_max_vi_per_vf - 1) % NVME_VF_RES_GRANULARITY) {
7874 error_setg(errp, "sriov_max_vi_per_vf must meet:"
7875 " (sriov_max_vi_per_vf - 1) %% %d == 0 and"
7876 " sriov_max_vi_per_vf >= 1", NVME_VF_RES_GRANULARITY);
7877 return false;
7878 }
7879
7880 if (params->sriov_max_vq_per_vf &&
7881 (params->sriov_max_vq_per_vf < 2 ||
7882 (params->sriov_max_vq_per_vf - 1) % NVME_VF_RES_GRANULARITY)) {
7883 error_setg(errp, "sriov_max_vq_per_vf must meet:"
7884 " (sriov_max_vq_per_vf - 1) %% %d == 0 and"
7885 " sriov_max_vq_per_vf >= 2", NVME_VF_RES_GRANULARITY);
7886 return false;
7887 }
7888 }
7889
7890 return true;
7891 }
7892
7893 static void nvme_init_state(NvmeCtrl *n)
7894 {
7895 NvmePriCtrlCap *cap = &n->pri_ctrl_cap;
7896 NvmeSecCtrlList *list = &n->sec_ctrl_list;
7897 NvmeSecCtrlEntry *sctrl;
7898 PCIDevice *pci = PCI_DEVICE(n);
7899 uint8_t max_vfs;
7900 int i;
7901
7902 if (pci_is_vf(pci)) {
7903 sctrl = nvme_sctrl(n);
7904 max_vfs = 0;
7905 n->conf_ioqpairs = sctrl->nvq ? le16_to_cpu(sctrl->nvq) - 1 : 0;
7906 n->conf_msix_qsize = sctrl->nvi ? le16_to_cpu(sctrl->nvi) : 1;
7907 } else {
7908 max_vfs = n->params.sriov_max_vfs;
7909 n->conf_ioqpairs = n->params.max_ioqpairs;
7910 n->conf_msix_qsize = n->params.msix_qsize;
7911 }
7912
7913 n->sq = g_new0(NvmeSQueue *, n->params.max_ioqpairs + 1);
7914 n->cq = g_new0(NvmeCQueue *, n->params.max_ioqpairs + 1);
7915 n->temperature = NVME_TEMPERATURE;
7916 n->features.temp_thresh_hi = NVME_TEMPERATURE_WARNING;
7917 n->starttime_ms = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
7918 n->aer_reqs = g_new0(NvmeRequest *, n->params.aerl + 1);
7919 QTAILQ_INIT(&n->aer_queue);
7920
7921 list->numcntl = cpu_to_le16(max_vfs);
7922 for (i = 0; i < max_vfs; i++) {
7923 sctrl = &list->sec[i];
7924 sctrl->pcid = cpu_to_le16(n->cntlid);
7925 sctrl->vfn = cpu_to_le16(i + 1);
7926 }
7927
7928 cap->cntlid = cpu_to_le16(n->cntlid);
7929 cap->crt = NVME_CRT_VQ | NVME_CRT_VI;
7930
7931 if (pci_is_vf(pci)) {
7932 cap->vqprt = cpu_to_le16(1 + n->conf_ioqpairs);
7933 } else {
7934 cap->vqprt = cpu_to_le16(1 + n->params.max_ioqpairs -
7935 n->params.sriov_vq_flexible);
7936 cap->vqfrt = cpu_to_le32(n->params.sriov_vq_flexible);
7937 cap->vqrfap = cap->vqfrt;
7938 cap->vqgran = cpu_to_le16(NVME_VF_RES_GRANULARITY);
7939 cap->vqfrsm = n->params.sriov_max_vq_per_vf ?
7940 cpu_to_le16(n->params.sriov_max_vq_per_vf) :
7941 cap->vqfrt / MAX(max_vfs, 1);
7942 }
7943
7944 if (pci_is_vf(pci)) {
7945 cap->viprt = cpu_to_le16(n->conf_msix_qsize);
7946 } else {
7947 cap->viprt = cpu_to_le16(n->params.msix_qsize -
7948 n->params.sriov_vi_flexible);
7949 cap->vifrt = cpu_to_le32(n->params.sriov_vi_flexible);
7950 cap->virfap = cap->vifrt;
7951 cap->vigran = cpu_to_le16(NVME_VF_RES_GRANULARITY);
7952 cap->vifrsm = n->params.sriov_max_vi_per_vf ?
7953 cpu_to_le16(n->params.sriov_max_vi_per_vf) :
7954 cap->vifrt / MAX(max_vfs, 1);
7955 }
7956 }
7957
7958 static void nvme_init_cmb(NvmeCtrl *n, PCIDevice *pci_dev)
7959 {
7960 uint64_t cmb_size = n->params.cmb_size_mb * MiB;
7961 uint64_t cap = ldq_le_p(&n->bar.cap);
7962
7963 n->cmb.buf = g_malloc0(cmb_size);
7964 memory_region_init_io(&n->cmb.mem, OBJECT(n), &nvme_cmb_ops, n,
7965 "nvme-cmb", cmb_size);
7966 pci_register_bar(pci_dev, NVME_CMB_BIR,
7967 PCI_BASE_ADDRESS_SPACE_MEMORY |
7968 PCI_BASE_ADDRESS_MEM_TYPE_64 |
7969 PCI_BASE_ADDRESS_MEM_PREFETCH, &n->cmb.mem);
7970
7971 NVME_CAP_SET_CMBS(cap, 1);
7972 stq_le_p(&n->bar.cap, cap);
7973
7974 if (n->params.legacy_cmb) {
7975 nvme_cmb_enable_regs(n);
7976 n->cmb.cmse = true;
7977 }
7978 }
7979
7980 static void nvme_init_pmr(NvmeCtrl *n, PCIDevice *pci_dev)
7981 {
7982 uint32_t pmrcap = ldl_le_p(&n->bar.pmrcap);
7983
7984 NVME_PMRCAP_SET_RDS(pmrcap, 1);
7985 NVME_PMRCAP_SET_WDS(pmrcap, 1);
7986 NVME_PMRCAP_SET_BIR(pmrcap, NVME_PMR_BIR);
7987 /* Turn on bit 1 support */
7988 NVME_PMRCAP_SET_PMRWBM(pmrcap, 0x02);
7989 NVME_PMRCAP_SET_CMSS(pmrcap, 1);
7990 stl_le_p(&n->bar.pmrcap, pmrcap);
7991
7992 pci_register_bar(pci_dev, NVME_PMR_BIR,
7993 PCI_BASE_ADDRESS_SPACE_MEMORY |
7994 PCI_BASE_ADDRESS_MEM_TYPE_64 |
7995 PCI_BASE_ADDRESS_MEM_PREFETCH, &n->pmr.dev->mr);
7996
7997 memory_region_set_enabled(&n->pmr.dev->mr, false);
7998 }
7999
8000 static uint64_t nvme_bar_size(unsigned total_queues, unsigned total_irqs,
8001 unsigned *msix_table_offset,
8002 unsigned *msix_pba_offset)
8003 {
8004 uint64_t bar_size, msix_table_size, msix_pba_size;
8005
8006 bar_size = sizeof(NvmeBar) + 2 * total_queues * NVME_DB_SIZE;
8007 bar_size = QEMU_ALIGN_UP(bar_size, 4 * KiB);
8008
8009 if (msix_table_offset) {
8010 *msix_table_offset = bar_size;
8011 }
8012
8013 msix_table_size = PCI_MSIX_ENTRY_SIZE * total_irqs;
8014 bar_size += msix_table_size;
8015 bar_size = QEMU_ALIGN_UP(bar_size, 4 * KiB);
8016
8017 if (msix_pba_offset) {
8018 *msix_pba_offset = bar_size;
8019 }
8020
8021 msix_pba_size = QEMU_ALIGN_UP(total_irqs, 64) / 8;
8022 bar_size += msix_pba_size;
8023
8024 bar_size = pow2ceil(bar_size);
8025 return bar_size;
8026 }
8027
8028 static void nvme_init_sriov(NvmeCtrl *n, PCIDevice *pci_dev, uint16_t offset)
8029 {
8030 uint16_t vf_dev_id = n->params.use_intel_id ?
8031 PCI_DEVICE_ID_INTEL_NVME : PCI_DEVICE_ID_REDHAT_NVME;
8032 NvmePriCtrlCap *cap = &n->pri_ctrl_cap;
8033 uint64_t bar_size = nvme_bar_size(le16_to_cpu(cap->vqfrsm),
8034 le16_to_cpu(cap->vifrsm),
8035 NULL, NULL);
8036
8037 pcie_sriov_pf_init(pci_dev, offset, "nvme", vf_dev_id,
8038 n->params.sriov_max_vfs, n->params.sriov_max_vfs,
8039 NVME_VF_OFFSET, NVME_VF_STRIDE);
8040
8041 pcie_sriov_pf_init_vf_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY |
8042 PCI_BASE_ADDRESS_MEM_TYPE_64, bar_size);
8043 }
8044
8045 static int nvme_add_pm_capability(PCIDevice *pci_dev, uint8_t offset)
8046 {
8047 Error *err = NULL;
8048 int ret;
8049
8050 ret = pci_add_capability(pci_dev, PCI_CAP_ID_PM, offset,
8051 PCI_PM_SIZEOF, &err);
8052 if (err) {
8053 error_report_err(err);
8054 return ret;
8055 }
8056
8057 pci_set_word(pci_dev->config + offset + PCI_PM_PMC,
8058 PCI_PM_CAP_VER_1_2);
8059 pci_set_word(pci_dev->config + offset + PCI_PM_CTRL,
8060 PCI_PM_CTRL_NO_SOFT_RESET);
8061 pci_set_word(pci_dev->wmask + offset + PCI_PM_CTRL,
8062 PCI_PM_CTRL_STATE_MASK);
8063
8064 return 0;
8065 }
8066
8067 static bool nvme_init_pci(NvmeCtrl *n, PCIDevice *pci_dev, Error **errp)
8068 {
8069 ERRP_GUARD();
8070 uint8_t *pci_conf = pci_dev->config;
8071 uint64_t bar_size;
8072 unsigned msix_table_offset, msix_pba_offset;
8073 int ret;
8074
8075 pci_conf[PCI_INTERRUPT_PIN] = 1;
8076 pci_config_set_prog_interface(pci_conf, 0x2);
8077
8078 if (n->params.use_intel_id) {
8079 pci_config_set_vendor_id(pci_conf, PCI_VENDOR_ID_INTEL);
8080 pci_config_set_device_id(pci_conf, PCI_DEVICE_ID_INTEL_NVME);
8081 } else {
8082 pci_config_set_vendor_id(pci_conf, PCI_VENDOR_ID_REDHAT);
8083 pci_config_set_device_id(pci_conf, PCI_DEVICE_ID_REDHAT_NVME);
8084 }
8085
8086 pci_config_set_class(pci_conf, PCI_CLASS_STORAGE_EXPRESS);
8087 nvme_add_pm_capability(pci_dev, 0x60);
8088 pcie_endpoint_cap_init(pci_dev, 0x80);
8089 pcie_cap_flr_init(pci_dev);
8090 if (n->params.sriov_max_vfs) {
8091 pcie_ari_init(pci_dev, 0x100, 1);
8092 }
8093
8094 /* add one to max_ioqpairs to account for the admin queue pair */
8095 bar_size = nvme_bar_size(n->params.max_ioqpairs + 1, n->params.msix_qsize,
8096 &msix_table_offset, &msix_pba_offset);
8097
8098 memory_region_init(&n->bar0, OBJECT(n), "nvme-bar0", bar_size);
8099 memory_region_init_io(&n->iomem, OBJECT(n), &nvme_mmio_ops, n, "nvme",
8100 msix_table_offset);
8101 memory_region_add_subregion(&n->bar0, 0, &n->iomem);
8102
8103 if (pci_is_vf(pci_dev)) {
8104 pcie_sriov_vf_register_bar(pci_dev, 0, &n->bar0);
8105 } else {
8106 pci_register_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY |
8107 PCI_BASE_ADDRESS_MEM_TYPE_64, &n->bar0);
8108 }
8109 ret = msix_init(pci_dev, n->params.msix_qsize,
8110 &n->bar0, 0, msix_table_offset,
8111 &n->bar0, 0, msix_pba_offset, 0, errp);
8112 if (ret == -ENOTSUP) {
8113 /* report that msix is not supported, but do not error out */
8114 warn_report_err(*errp);
8115 *errp = NULL;
8116 } else if (ret < 0) {
8117 /* propagate error to caller */
8118 return false;
8119 }
8120
8121 nvme_update_msixcap_ts(pci_dev, n->conf_msix_qsize);
8122
8123 if (n->params.cmb_size_mb) {
8124 nvme_init_cmb(n, pci_dev);
8125 }
8126
8127 if (n->pmr.dev) {
8128 nvme_init_pmr(n, pci_dev);
8129 }
8130
8131 if (!pci_is_vf(pci_dev) && n->params.sriov_max_vfs) {
8132 nvme_init_sriov(n, pci_dev, 0x120);
8133 }
8134
8135 return true;
8136 }
8137
8138 static void nvme_init_subnqn(NvmeCtrl *n)
8139 {
8140 NvmeSubsystem *subsys = n->subsys;
8141 NvmeIdCtrl *id = &n->id_ctrl;
8142
8143 if (!subsys) {
8144 snprintf((char *)id->subnqn, sizeof(id->subnqn),
8145 "nqn.2019-08.org.qemu:%s", n->params.serial);
8146 } else {
8147 pstrcpy((char *)id->subnqn, sizeof(id->subnqn), (char*)subsys->subnqn);
8148 }
8149 }
8150
8151 static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev)
8152 {
8153 NvmeIdCtrl *id = &n->id_ctrl;
8154 uint8_t *pci_conf = pci_dev->config;
8155 uint64_t cap = ldq_le_p(&n->bar.cap);
8156 NvmeSecCtrlEntry *sctrl = nvme_sctrl(n);
8157 uint32_t ctratt;
8158
8159 id->vid = cpu_to_le16(pci_get_word(pci_conf + PCI_VENDOR_ID));
8160 id->ssvid = cpu_to_le16(pci_get_word(pci_conf + PCI_SUBSYSTEM_VENDOR_ID));
8161 strpadcpy((char *)id->mn, sizeof(id->mn), "QEMU NVMe Ctrl", ' ');
8162 strpadcpy((char *)id->fr, sizeof(id->fr), QEMU_VERSION, ' ');
8163 strpadcpy((char *)id->sn, sizeof(id->sn), n->params.serial, ' ');
8164
8165 id->cntlid = cpu_to_le16(n->cntlid);
8166
8167 id->oaes = cpu_to_le32(NVME_OAES_NS_ATTR);
8168 ctratt = NVME_CTRATT_ELBAS;
8169
8170 id->rab = 6;
8171
8172 if (n->params.use_intel_id) {
8173 id->ieee[0] = 0xb3;
8174 id->ieee[1] = 0x02;
8175 id->ieee[2] = 0x00;
8176 } else {
8177 id->ieee[0] = 0x00;
8178 id->ieee[1] = 0x54;
8179 id->ieee[2] = 0x52;
8180 }
8181
8182 id->mdts = n->params.mdts;
8183 id->ver = cpu_to_le32(NVME_SPEC_VER);
8184 id->oacs =
8185 cpu_to_le16(NVME_OACS_NS_MGMT | NVME_OACS_FORMAT | NVME_OACS_DBBUF |
8186 NVME_OACS_DIRECTIVES);
8187 id->cntrltype = 0x1;
8188
8189 /*
8190 * Because the controller always completes the Abort command immediately,
8191 * there can never be more than one concurrently executing Abort command,
8192 * so this value is never used for anything. Note that there can easily be
8193 * many Abort commands in the queues, but they are not considered
8194 * "executing" until processed by nvme_abort.
8195 *
8196 * The specification recommends a value of 3 for Abort Command Limit (four
8197 * concurrently outstanding Abort commands), so lets use that though it is
8198 * inconsequential.
8199 */
8200 id->acl = 3;
8201 id->aerl = n->params.aerl;
8202 id->frmw = (NVME_NUM_FW_SLOTS << 1) | NVME_FRMW_SLOT1_RO;
8203 id->lpa = NVME_LPA_NS_SMART | NVME_LPA_CSE | NVME_LPA_EXTENDED;
8204
8205 /* recommended default value (~70 C) */
8206 id->wctemp = cpu_to_le16(NVME_TEMPERATURE_WARNING);
8207 id->cctemp = cpu_to_le16(NVME_TEMPERATURE_CRITICAL);
8208
8209 id->sqes = (0x6 << 4) | 0x6;
8210 id->cqes = (0x4 << 4) | 0x4;
8211 id->nn = cpu_to_le32(NVME_MAX_NAMESPACES);
8212 id->oncs = cpu_to_le16(NVME_ONCS_WRITE_ZEROES | NVME_ONCS_TIMESTAMP |
8213 NVME_ONCS_FEATURES | NVME_ONCS_DSM |
8214 NVME_ONCS_COMPARE | NVME_ONCS_COPY);
8215
8216 /*
8217 * NOTE: If this device ever supports a command set that does NOT use 0x0
8218 * as a Flush-equivalent operation, support for the broadcast NSID in Flush
8219 * should probably be removed.
8220 *
8221 * See comment in nvme_io_cmd.
8222 */
8223 id->vwc = NVME_VWC_NSID_BROADCAST_SUPPORT | NVME_VWC_PRESENT;
8224
8225 id->ocfs = cpu_to_le16(NVME_OCFS_COPY_FORMAT_0 | NVME_OCFS_COPY_FORMAT_1);
8226 id->sgls = cpu_to_le32(NVME_CTRL_SGLS_SUPPORT_NO_ALIGN);
8227
8228 nvme_init_subnqn(n);
8229
8230 id->psd[0].mp = cpu_to_le16(0x9c4);
8231 id->psd[0].enlat = cpu_to_le32(0x10);
8232 id->psd[0].exlat = cpu_to_le32(0x4);
8233
8234 if (n->subsys) {
8235 id->cmic |= NVME_CMIC_MULTI_CTRL;
8236 ctratt |= NVME_CTRATT_ENDGRPS;
8237
8238 id->endgidmax = cpu_to_le16(0x1);
8239
8240 if (n->subsys->endgrp.fdp.enabled) {
8241 ctratt |= NVME_CTRATT_FDPS;
8242 }
8243 }
8244
8245 id->ctratt = cpu_to_le32(ctratt);
8246
8247 NVME_CAP_SET_MQES(cap, 0x7ff);
8248 NVME_CAP_SET_CQR(cap, 1);
8249 NVME_CAP_SET_TO(cap, 0xf);
8250 NVME_CAP_SET_CSS(cap, NVME_CAP_CSS_NVM);
8251 NVME_CAP_SET_CSS(cap, NVME_CAP_CSS_CSI_SUPP);
8252 NVME_CAP_SET_CSS(cap, NVME_CAP_CSS_ADMIN_ONLY);
8253 NVME_CAP_SET_MPSMAX(cap, 4);
8254 NVME_CAP_SET_CMBS(cap, n->params.cmb_size_mb ? 1 : 0);
8255 NVME_CAP_SET_PMRS(cap, n->pmr.dev ? 1 : 0);
8256 stq_le_p(&n->bar.cap, cap);
8257
8258 stl_le_p(&n->bar.vs, NVME_SPEC_VER);
8259 n->bar.intmc = n->bar.intms = 0;
8260
8261 if (pci_is_vf(pci_dev) && !sctrl->scs) {
8262 stl_le_p(&n->bar.csts, NVME_CSTS_FAILED);
8263 }
8264 }
8265
8266 static int nvme_init_subsys(NvmeCtrl *n, Error **errp)
8267 {
8268 int cntlid;
8269
8270 if (!n->subsys) {
8271 return 0;
8272 }
8273
8274 cntlid = nvme_subsys_register_ctrl(n, errp);
8275 if (cntlid < 0) {
8276 return -1;
8277 }
8278
8279 n->cntlid = cntlid;
8280
8281 return 0;
8282 }
8283
8284 void nvme_attach_ns(NvmeCtrl *n, NvmeNamespace *ns)
8285 {
8286 uint32_t nsid = ns->params.nsid;
8287 assert(nsid && nsid <= NVME_MAX_NAMESPACES);
8288
8289 n->namespaces[nsid] = ns;
8290 ns->attached++;
8291
8292 n->dmrsl = MIN_NON_ZERO(n->dmrsl,
8293 BDRV_REQUEST_MAX_BYTES / nvme_l2b(ns, 1));
8294 }
8295
8296 static void nvme_realize(PCIDevice *pci_dev, Error **errp)
8297 {
8298 NvmeCtrl *n = NVME(pci_dev);
8299 DeviceState *dev = DEVICE(pci_dev);
8300 NvmeNamespace *ns;
8301 NvmeCtrl *pn = NVME(pcie_sriov_get_pf(pci_dev));
8302
8303 if (pci_is_vf(pci_dev)) {
8304 /*
8305 * VFs derive settings from the parent. PF's lifespan exceeds
8306 * that of VF's, so it's safe to share params.serial.
8307 */
8308 memcpy(&n->params, &pn->params, sizeof(NvmeParams));
8309 n->subsys = pn->subsys;
8310 }
8311
8312 if (!nvme_check_params(n, errp)) {
8313 return;
8314 }
8315
8316 qbus_init(&n->bus, sizeof(NvmeBus), TYPE_NVME_BUS, dev, dev->id);
8317
8318 if (nvme_init_subsys(n, errp)) {
8319 return;
8320 }
8321 nvme_init_state(n);
8322 if (!nvme_init_pci(n, pci_dev, errp)) {
8323 return;
8324 }
8325 nvme_init_ctrl(n, pci_dev);
8326
8327 /* setup a namespace if the controller drive property was given */
8328 if (n->namespace.blkconf.blk) {
8329 ns = &n->namespace;
8330 ns->params.nsid = 1;
8331
8332 if (nvme_ns_setup(ns, errp)) {
8333 return;
8334 }
8335
8336 nvme_attach_ns(n, ns);
8337 }
8338 }
8339
8340 static void nvme_exit(PCIDevice *pci_dev)
8341 {
8342 NvmeCtrl *n = NVME(pci_dev);
8343 NvmeNamespace *ns;
8344 int i;
8345
8346 nvme_ctrl_reset(n, NVME_RESET_FUNCTION);
8347
8348 if (n->subsys) {
8349 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
8350 ns = nvme_ns(n, i);
8351 if (ns) {
8352 ns->attached--;
8353 }
8354 }
8355
8356 nvme_subsys_unregister_ctrl(n->subsys, n);
8357 }
8358
8359 g_free(n->cq);
8360 g_free(n->sq);
8361 g_free(n->aer_reqs);
8362
8363 if (n->params.cmb_size_mb) {
8364 g_free(n->cmb.buf);
8365 }
8366
8367 if (n->pmr.dev) {
8368 host_memory_backend_set_mapped(n->pmr.dev, false);
8369 }
8370
8371 if (!pci_is_vf(pci_dev) && n->params.sriov_max_vfs) {
8372 pcie_sriov_pf_exit(pci_dev);
8373 }
8374
8375 msix_uninit(pci_dev, &n->bar0, &n->bar0);
8376 memory_region_del_subregion(&n->bar0, &n->iomem);
8377 }
8378
8379 static Property nvme_props[] = {
8380 DEFINE_BLOCK_PROPERTIES(NvmeCtrl, namespace.blkconf),
8381 DEFINE_PROP_LINK("pmrdev", NvmeCtrl, pmr.dev, TYPE_MEMORY_BACKEND,
8382 HostMemoryBackend *),
8383 DEFINE_PROP_LINK("subsys", NvmeCtrl, subsys, TYPE_NVME_SUBSYS,
8384 NvmeSubsystem *),
8385 DEFINE_PROP_STRING("serial", NvmeCtrl, params.serial),
8386 DEFINE_PROP_UINT32("cmb_size_mb", NvmeCtrl, params.cmb_size_mb, 0),
8387 DEFINE_PROP_UINT32("num_queues", NvmeCtrl, params.num_queues, 0),
8388 DEFINE_PROP_UINT32("max_ioqpairs", NvmeCtrl, params.max_ioqpairs, 64),
8389 DEFINE_PROP_UINT16("msix_qsize", NvmeCtrl, params.msix_qsize, 65),
8390 DEFINE_PROP_UINT8("aerl", NvmeCtrl, params.aerl, 3),
8391 DEFINE_PROP_UINT32("aer_max_queued", NvmeCtrl, params.aer_max_queued, 64),
8392 DEFINE_PROP_UINT8("mdts", NvmeCtrl, params.mdts, 7),
8393 DEFINE_PROP_UINT8("vsl", NvmeCtrl, params.vsl, 7),
8394 DEFINE_PROP_BOOL("use-intel-id", NvmeCtrl, params.use_intel_id, false),
8395 DEFINE_PROP_BOOL("legacy-cmb", NvmeCtrl, params.legacy_cmb, false),
8396 DEFINE_PROP_BOOL("ioeventfd", NvmeCtrl, params.ioeventfd, false),
8397 DEFINE_PROP_UINT8("zoned.zasl", NvmeCtrl, params.zasl, 0),
8398 DEFINE_PROP_BOOL("zoned.auto_transition", NvmeCtrl,
8399 params.auto_transition_zones, true),
8400 DEFINE_PROP_UINT8("sriov_max_vfs", NvmeCtrl, params.sriov_max_vfs, 0),
8401 DEFINE_PROP_UINT16("sriov_vq_flexible", NvmeCtrl,
8402 params.sriov_vq_flexible, 0),
8403 DEFINE_PROP_UINT16("sriov_vi_flexible", NvmeCtrl,
8404 params.sriov_vi_flexible, 0),
8405 DEFINE_PROP_UINT8("sriov_max_vi_per_vf", NvmeCtrl,
8406 params.sriov_max_vi_per_vf, 0),
8407 DEFINE_PROP_UINT8("sriov_max_vq_per_vf", NvmeCtrl,
8408 params.sriov_max_vq_per_vf, 0),
8409 DEFINE_PROP_END_OF_LIST(),
8410 };
8411
8412 static void nvme_get_smart_warning(Object *obj, Visitor *v, const char *name,
8413 void *opaque, Error **errp)
8414 {
8415 NvmeCtrl *n = NVME(obj);
8416 uint8_t value = n->smart_critical_warning;
8417
8418 visit_type_uint8(v, name, &value, errp);
8419 }
8420
8421 static void nvme_set_smart_warning(Object *obj, Visitor *v, const char *name,
8422 void *opaque, Error **errp)
8423 {
8424 NvmeCtrl *n = NVME(obj);
8425 uint8_t value, old_value, cap = 0, index, event;
8426
8427 if (!visit_type_uint8(v, name, &value, errp)) {
8428 return;
8429 }
8430
8431 cap = NVME_SMART_SPARE | NVME_SMART_TEMPERATURE | NVME_SMART_RELIABILITY
8432 | NVME_SMART_MEDIA_READ_ONLY | NVME_SMART_FAILED_VOLATILE_MEDIA;
8433 if (NVME_CAP_PMRS(ldq_le_p(&n->bar.cap))) {
8434 cap |= NVME_SMART_PMR_UNRELIABLE;
8435 }
8436
8437 if ((value & cap) != value) {
8438 error_setg(errp, "unsupported smart critical warning bits: 0x%x",
8439 value & ~cap);
8440 return;
8441 }
8442
8443 old_value = n->smart_critical_warning;
8444 n->smart_critical_warning = value;
8445
8446 /* only inject new bits of smart critical warning */
8447 for (index = 0; index < NVME_SMART_WARN_MAX; index++) {
8448 event = 1 << index;
8449 if (value & ~old_value & event)
8450 nvme_smart_event(n, event);
8451 }
8452 }
8453
8454 static void nvme_pci_reset(DeviceState *qdev)
8455 {
8456 PCIDevice *pci_dev = PCI_DEVICE(qdev);
8457 NvmeCtrl *n = NVME(pci_dev);
8458
8459 trace_pci_nvme_pci_reset();
8460 nvme_ctrl_reset(n, NVME_RESET_FUNCTION);
8461 }
8462
8463 static void nvme_sriov_pre_write_ctrl(PCIDevice *dev, uint32_t address,
8464 uint32_t val, int len)
8465 {
8466 NvmeCtrl *n = NVME(dev);
8467 NvmeSecCtrlEntry *sctrl;
8468 uint16_t sriov_cap = dev->exp.sriov_cap;
8469 uint32_t off = address - sriov_cap;
8470 int i, num_vfs;
8471
8472 if (!sriov_cap) {
8473 return;
8474 }
8475
8476 if (range_covers_byte(off, len, PCI_SRIOV_CTRL)) {
8477 if (!(val & PCI_SRIOV_CTRL_VFE)) {
8478 num_vfs = pci_get_word(dev->config + sriov_cap + PCI_SRIOV_NUM_VF);
8479 for (i = 0; i < num_vfs; i++) {
8480 sctrl = &n->sec_ctrl_list.sec[i];
8481 nvme_virt_set_state(n, le16_to_cpu(sctrl->scid), false);
8482 }
8483 }
8484 }
8485 }
8486
8487 static void nvme_pci_write_config(PCIDevice *dev, uint32_t address,
8488 uint32_t val, int len)
8489 {
8490 nvme_sriov_pre_write_ctrl(dev, address, val, len);
8491 pci_default_write_config(dev, address, val, len);
8492 pcie_cap_flr_write_config(dev, address, val, len);
8493 }
8494
8495 static const VMStateDescription nvme_vmstate = {
8496 .name = "nvme",
8497 .unmigratable = 1,
8498 };
8499
8500 static void nvme_class_init(ObjectClass *oc, void *data)
8501 {
8502 DeviceClass *dc = DEVICE_CLASS(oc);
8503 PCIDeviceClass *pc = PCI_DEVICE_CLASS(oc);
8504
8505 pc->realize = nvme_realize;
8506 pc->config_write = nvme_pci_write_config;
8507 pc->exit = nvme_exit;
8508 pc->class_id = PCI_CLASS_STORAGE_EXPRESS;
8509 pc->revision = 2;
8510
8511 set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
8512 dc->desc = "Non-Volatile Memory Express";
8513 device_class_set_props(dc, nvme_props);
8514 dc->vmsd = &nvme_vmstate;
8515 dc->reset = nvme_pci_reset;
8516 }
8517
8518 static void nvme_instance_init(Object *obj)
8519 {
8520 NvmeCtrl *n = NVME(obj);
8521
8522 device_add_bootindex_property(obj, &n->namespace.blkconf.bootindex,
8523 "bootindex", "/namespace@1,0",
8524 DEVICE(obj));
8525
8526 object_property_add(obj, "smart_critical_warning", "uint8",
8527 nvme_get_smart_warning,
8528 nvme_set_smart_warning, NULL, NULL);
8529 }
8530
8531 static const TypeInfo nvme_info = {
8532 .name = TYPE_NVME,
8533 .parent = TYPE_PCI_DEVICE,
8534 .instance_size = sizeof(NvmeCtrl),
8535 .instance_init = nvme_instance_init,
8536 .class_init = nvme_class_init,
8537 .interfaces = (InterfaceInfo[]) {
8538 { INTERFACE_PCIE_DEVICE },
8539 { }
8540 },
8541 };
8542
8543 static const TypeInfo nvme_bus_info = {
8544 .name = TYPE_NVME_BUS,
8545 .parent = TYPE_BUS,
8546 .instance_size = sizeof(NvmeBus),
8547 };
8548
8549 static void nvme_register_types(void)
8550 {
8551 type_register_static(&nvme_info);
8552 type_register_static(&nvme_bus_info);
8553 }
8554
8555 type_init(nvme_register_types)
QEMU mirror