2 * QEMU NVM Express Controller
4 * Copyright (c) 2012, Intel Corporation
6 * Written by Keith Busch <keith.busch@intel.com>
8 * This code is licensed under the GNU GPL v2 or later.
12 * Reference Specs: http://www.nvmexpress.org, 1.2, 1.1, 1.0e
14 * https://nvmexpress.org/developers/nvme-specification/
19 * -drive file=<file>,if=none,id=<drive_id>
20 * -device nvme,drive=<drive_id>,serial=<serial>,id=<id[optional]>, \
21 * cmb_size_mb=<cmb_size_mb[optional]>, \
22 * [pmrdev=<mem_backend_file_id>,] \
23 * max_ioqpairs=<N[optional]>, \
24 * aerl=<N[optional]>, aer_max_queued=<N[optional]>
26 * Note cmb_size_mb denotes size of CMB in MB. CMB is assumed to be at
27 * offset 0 in BAR2 and supports only WDS, RDS and SQS for now.
29 * cmb_size_mb= and pmrdev= options are mutually exclusive due to limitation
30 * in available BAR's. cmb_size_mb= will take precedence over pmrdev= when
32 * Enabling pmr emulation can be achieved by pointing to memory-backend-file.
34 * -object memory-backend-file,id=<mem_id>,share=on,mem-path=<file_path>, \
35 * size=<size> .... -device nvme,...,pmrdev=<mem_id>
38 * nvme device parameters
39 * ~~~~~~~~~~~~~~~~~~~~~~
41 * The Asynchronous Event Request Limit (AERL). Indicates the maximum number
42 * of concurrently outstanding Asynchronous Event Request commands suppoert
43 * by the controller. This is a 0's based value.
46 * This is the maximum number of events that the device will enqueue for
47 * completion when there are no oustanding AERs. When the maximum number of
48 * enqueued events are reached, subsequent events will be dropped.
52 #include "qemu/osdep.h"
53 #include "qemu/units.h"
54 #include "qemu/error-report.h"
55 #include "hw/block/block.h"
56 #include "hw/pci/msix.h"
57 #include "hw/pci/pci.h"
58 #include "hw/qdev-properties.h"
59 #include "migration/vmstate.h"
60 #include "sysemu/sysemu.h"
61 #include "qapi/error.h"
62 #include "qapi/visitor.h"
63 #include "sysemu/hostmem.h"
64 #include "sysemu/block-backend.h"
65 #include "exec/memory.h"
67 #include "qemu/module.h"
68 #include "qemu/cutils.h"
72 #define NVME_MAX_IOQPAIRS 0xffff
73 #define NVME_DB_SIZE 4
74 #define NVME_CMB_BIR 2
75 #define NVME_PMR_BIR 2
76 #define NVME_TEMPERATURE 0x143
77 #define NVME_TEMPERATURE_WARNING 0x157
78 #define NVME_TEMPERATURE_CRITICAL 0x175
79 #define NVME_NUM_FW_SLOTS 1
81 #define NVME_GUEST_ERR(trace, fmt, ...) \
83 (trace_##trace)(__VA_ARGS__); \
84 qemu_log_mask(LOG_GUEST_ERROR, #trace \
85 " in %s: " fmt "\n", __func__, ## __VA_ARGS__); \
88 static const bool nvme_feature_support
[NVME_FID_MAX
] = {
89 [NVME_ARBITRATION
] = true,
90 [NVME_POWER_MANAGEMENT
] = true,
91 [NVME_TEMPERATURE_THRESHOLD
] = true,
92 [NVME_ERROR_RECOVERY
] = true,
93 [NVME_VOLATILE_WRITE_CACHE
] = true,
94 [NVME_NUMBER_OF_QUEUES
] = true,
95 [NVME_INTERRUPT_COALESCING
] = true,
96 [NVME_INTERRUPT_VECTOR_CONF
] = true,
97 [NVME_WRITE_ATOMICITY
] = true,
98 [NVME_ASYNCHRONOUS_EVENT_CONF
] = true,
99 [NVME_TIMESTAMP
] = true,
102 static const uint32_t nvme_feature_cap
[NVME_FID_MAX
] = {
103 [NVME_TEMPERATURE_THRESHOLD
] = NVME_FEAT_CAP_CHANGE
,
104 [NVME_VOLATILE_WRITE_CACHE
] = NVME_FEAT_CAP_CHANGE
,
105 [NVME_NUMBER_OF_QUEUES
] = NVME_FEAT_CAP_CHANGE
,
106 [NVME_ASYNCHRONOUS_EVENT_CONF
] = NVME_FEAT_CAP_CHANGE
,
107 [NVME_TIMESTAMP
] = NVME_FEAT_CAP_CHANGE
,
110 static void nvme_process_sq(void *opaque
);
112 static uint16_t nvme_cid(NvmeRequest
*req
)
118 return le16_to_cpu(req
->cqe
.cid
);
121 static uint16_t nvme_sqid(NvmeRequest
*req
)
123 return le16_to_cpu(req
->sq
->sqid
);
126 static bool nvme_addr_is_cmb(NvmeCtrl
*n
, hwaddr addr
)
128 hwaddr low
= n
->ctrl_mem
.addr
;
129 hwaddr hi
= n
->ctrl_mem
.addr
+ int128_get64(n
->ctrl_mem
.size
);
131 return addr
>= low
&& addr
< hi
;
134 static void nvme_addr_read(NvmeCtrl
*n
, hwaddr addr
, void *buf
, int size
)
136 if (n
->bar
.cmbsz
&& nvme_addr_is_cmb(n
, addr
)) {
137 memcpy(buf
, (void *)&n
->cmbuf
[addr
- n
->ctrl_mem
.addr
], size
);
141 pci_dma_read(&n
->parent_obj
, addr
, buf
, size
);
144 static int nvme_check_sqid(NvmeCtrl
*n
, uint16_t sqid
)
146 return sqid
< n
->params
.max_ioqpairs
+ 1 && n
->sq
[sqid
] != NULL
? 0 : -1;
149 static int nvme_check_cqid(NvmeCtrl
*n
, uint16_t cqid
)
151 return cqid
< n
->params
.max_ioqpairs
+ 1 && n
->cq
[cqid
] != NULL
? 0 : -1;
154 static void nvme_inc_cq_tail(NvmeCQueue
*cq
)
157 if (cq
->tail
>= cq
->size
) {
159 cq
->phase
= !cq
->phase
;
163 static void nvme_inc_sq_head(NvmeSQueue
*sq
)
165 sq
->head
= (sq
->head
+ 1) % sq
->size
;
168 static uint8_t nvme_cq_full(NvmeCQueue
*cq
)
170 return (cq
->tail
+ 1) % cq
->size
== cq
->head
;
173 static uint8_t nvme_sq_empty(NvmeSQueue
*sq
)
175 return sq
->head
== sq
->tail
;
178 static void nvme_irq_check(NvmeCtrl
*n
)
180 if (msix_enabled(&(n
->parent_obj
))) {
183 if (~n
->bar
.intms
& n
->irq_status
) {
184 pci_irq_assert(&n
->parent_obj
);
186 pci_irq_deassert(&n
->parent_obj
);
190 static void nvme_irq_assert(NvmeCtrl
*n
, NvmeCQueue
*cq
)
192 if (cq
->irq_enabled
) {
193 if (msix_enabled(&(n
->parent_obj
))) {
194 trace_pci_nvme_irq_msix(cq
->vector
);
195 msix_notify(&(n
->parent_obj
), cq
->vector
);
197 trace_pci_nvme_irq_pin();
198 assert(cq
->vector
< 32);
199 n
->irq_status
|= 1 << cq
->vector
;
203 trace_pci_nvme_irq_masked();
207 static void nvme_irq_deassert(NvmeCtrl
*n
, NvmeCQueue
*cq
)
209 if (cq
->irq_enabled
) {
210 if (msix_enabled(&(n
->parent_obj
))) {
213 assert(cq
->vector
< 32);
214 n
->irq_status
&= ~(1 << cq
->vector
);
220 static uint16_t nvme_map_prp(QEMUSGList
*qsg
, QEMUIOVector
*iov
, uint64_t prp1
,
221 uint64_t prp2
, uint32_t len
, NvmeCtrl
*n
)
223 hwaddr trans_len
= n
->page_size
- (prp1
% n
->page_size
);
224 trans_len
= MIN(len
, trans_len
);
225 int num_prps
= (len
>> n
->page_bits
) + 1;
227 if (unlikely(!prp1
)) {
228 trace_pci_nvme_err_invalid_prp();
229 return NVME_INVALID_FIELD
| NVME_DNR
;
230 } else if (n
->bar
.cmbsz
&& prp1
>= n
->ctrl_mem
.addr
&&
231 prp1
< n
->ctrl_mem
.addr
+ int128_get64(n
->ctrl_mem
.size
)) {
233 qemu_iovec_init(iov
, num_prps
);
234 qemu_iovec_add(iov
, (void *)&n
->cmbuf
[prp1
- n
->ctrl_mem
.addr
], trans_len
);
236 pci_dma_sglist_init(qsg
, &n
->parent_obj
, num_prps
);
237 qemu_sglist_add(qsg
, prp1
, trans_len
);
241 if (unlikely(!prp2
)) {
242 trace_pci_nvme_err_invalid_prp2_missing();
245 if (len
> n
->page_size
) {
246 uint64_t prp_list
[n
->max_prp_ents
];
247 uint32_t nents
, prp_trans
;
250 nents
= (len
+ n
->page_size
- 1) >> n
->page_bits
;
251 prp_trans
= MIN(n
->max_prp_ents
, nents
) * sizeof(uint64_t);
252 nvme_addr_read(n
, prp2
, (void *)prp_list
, prp_trans
);
254 uint64_t prp_ent
= le64_to_cpu(prp_list
[i
]);
256 if (i
== n
->max_prp_ents
- 1 && len
> n
->page_size
) {
257 if (unlikely(!prp_ent
|| prp_ent
& (n
->page_size
- 1))) {
258 trace_pci_nvme_err_invalid_prplist_ent(prp_ent
);
263 nents
= (len
+ n
->page_size
- 1) >> n
->page_bits
;
264 prp_trans
= MIN(n
->max_prp_ents
, nents
) * sizeof(uint64_t);
265 nvme_addr_read(n
, prp_ent
, (void *)prp_list
,
267 prp_ent
= le64_to_cpu(prp_list
[i
]);
270 if (unlikely(!prp_ent
|| prp_ent
& (n
->page_size
- 1))) {
271 trace_pci_nvme_err_invalid_prplist_ent(prp_ent
);
275 trans_len
= MIN(len
, n
->page_size
);
277 qemu_sglist_add(qsg
, prp_ent
, trans_len
);
279 qemu_iovec_add(iov
, (void *)&n
->cmbuf
[prp_ent
- n
->ctrl_mem
.addr
], trans_len
);
285 if (unlikely(prp2
& (n
->page_size
- 1))) {
286 trace_pci_nvme_err_invalid_prp2_align(prp2
);
290 qemu_sglist_add(qsg
, prp2
, len
);
292 qemu_iovec_add(iov
, (void *)&n
->cmbuf
[prp2
- n
->ctrl_mem
.addr
], trans_len
);
299 qemu_sglist_destroy(qsg
);
300 return NVME_INVALID_FIELD
| NVME_DNR
;
303 static uint16_t nvme_dma_write_prp(NvmeCtrl
*n
, uint8_t *ptr
, uint32_t len
,
304 uint64_t prp1
, uint64_t prp2
)
308 uint16_t status
= NVME_SUCCESS
;
310 if (nvme_map_prp(&qsg
, &iov
, prp1
, prp2
, len
, n
)) {
311 return NVME_INVALID_FIELD
| NVME_DNR
;
314 if (dma_buf_write(ptr
, len
, &qsg
)) {
315 status
= NVME_INVALID_FIELD
| NVME_DNR
;
317 qemu_sglist_destroy(&qsg
);
319 if (qemu_iovec_to_buf(&iov
, 0, ptr
, len
) != len
) {
320 status
= NVME_INVALID_FIELD
| NVME_DNR
;
322 qemu_iovec_destroy(&iov
);
327 static uint16_t nvme_dma_read_prp(NvmeCtrl
*n
, uint8_t *ptr
, uint32_t len
,
328 uint64_t prp1
, uint64_t prp2
)
332 uint16_t status
= NVME_SUCCESS
;
334 trace_pci_nvme_dma_read(prp1
, prp2
);
336 if (nvme_map_prp(&qsg
, &iov
, prp1
, prp2
, len
, n
)) {
337 return NVME_INVALID_FIELD
| NVME_DNR
;
340 if (unlikely(dma_buf_read(ptr
, len
, &qsg
))) {
341 trace_pci_nvme_err_invalid_dma();
342 status
= NVME_INVALID_FIELD
| NVME_DNR
;
344 qemu_sglist_destroy(&qsg
);
346 if (unlikely(qemu_iovec_from_buf(&iov
, 0, ptr
, len
) != len
)) {
347 trace_pci_nvme_err_invalid_dma();
348 status
= NVME_INVALID_FIELD
| NVME_DNR
;
350 qemu_iovec_destroy(&iov
);
355 static void nvme_post_cqes(void *opaque
)
357 NvmeCQueue
*cq
= opaque
;
358 NvmeCtrl
*n
= cq
->ctrl
;
359 NvmeRequest
*req
, *next
;
361 QTAILQ_FOREACH_SAFE(req
, &cq
->req_list
, entry
, next
) {
365 if (nvme_cq_full(cq
)) {
369 QTAILQ_REMOVE(&cq
->req_list
, req
, entry
);
371 req
->cqe
.status
= cpu_to_le16((req
->status
<< 1) | cq
->phase
);
372 req
->cqe
.sq_id
= cpu_to_le16(sq
->sqid
);
373 req
->cqe
.sq_head
= cpu_to_le16(sq
->head
);
374 addr
= cq
->dma_addr
+ cq
->tail
* n
->cqe_size
;
375 nvme_inc_cq_tail(cq
);
376 pci_dma_write(&n
->parent_obj
, addr
, (void *)&req
->cqe
,
378 QTAILQ_INSERT_TAIL(&sq
->req_list
, req
, entry
);
380 if (cq
->tail
!= cq
->head
) {
381 nvme_irq_assert(n
, cq
);
385 static void nvme_enqueue_req_completion(NvmeCQueue
*cq
, NvmeRequest
*req
)
387 assert(cq
->cqid
== req
->sq
->cqid
);
388 trace_pci_nvme_enqueue_req_completion(nvme_cid(req
), cq
->cqid
,
390 QTAILQ_REMOVE(&req
->sq
->out_req_list
, req
, entry
);
391 QTAILQ_INSERT_TAIL(&cq
->req_list
, req
, entry
);
392 timer_mod(cq
->timer
, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL
) + 500);
395 static void nvme_process_aers(void *opaque
)
397 NvmeCtrl
*n
= opaque
;
398 NvmeAsyncEvent
*event
, *next
;
400 trace_pci_nvme_process_aers(n
->aer_queued
);
402 QTAILQ_FOREACH_SAFE(event
, &n
->aer_queue
, entry
, next
) {
404 NvmeAerResult
*result
;
406 /* can't post cqe if there is nothing to complete */
407 if (!n
->outstanding_aers
) {
408 trace_pci_nvme_no_outstanding_aers();
412 /* ignore if masked (cqe posted, but event not cleared) */
413 if (n
->aer_mask
& (1 << event
->result
.event_type
)) {
414 trace_pci_nvme_aer_masked(event
->result
.event_type
, n
->aer_mask
);
418 QTAILQ_REMOVE(&n
->aer_queue
, event
, entry
);
421 n
->aer_mask
|= 1 << event
->result
.event_type
;
422 n
->outstanding_aers
--;
424 req
= n
->aer_reqs
[n
->outstanding_aers
];
426 result
= (NvmeAerResult
*) &req
->cqe
.result
;
427 result
->event_type
= event
->result
.event_type
;
428 result
->event_info
= event
->result
.event_info
;
429 result
->log_page
= event
->result
.log_page
;
432 req
->status
= NVME_SUCCESS
;
434 trace_pci_nvme_aer_post_cqe(result
->event_type
, result
->event_info
,
437 nvme_enqueue_req_completion(&n
->admin_cq
, req
);
441 static void nvme_enqueue_event(NvmeCtrl
*n
, uint8_t event_type
,
442 uint8_t event_info
, uint8_t log_page
)
444 NvmeAsyncEvent
*event
;
446 trace_pci_nvme_enqueue_event(event_type
, event_info
, log_page
);
448 if (n
->aer_queued
== n
->params
.aer_max_queued
) {
449 trace_pci_nvme_enqueue_event_noqueue(n
->aer_queued
);
453 event
= g_new(NvmeAsyncEvent
, 1);
454 event
->result
= (NvmeAerResult
) {
455 .event_type
= event_type
,
456 .event_info
= event_info
,
457 .log_page
= log_page
,
460 QTAILQ_INSERT_TAIL(&n
->aer_queue
, event
, entry
);
463 nvme_process_aers(n
);
466 static void nvme_clear_events(NvmeCtrl
*n
, uint8_t event_type
)
468 n
->aer_mask
&= ~(1 << event_type
);
469 if (!QTAILQ_EMPTY(&n
->aer_queue
)) {
470 nvme_process_aers(n
);
474 static void nvme_rw_cb(void *opaque
, int ret
)
476 NvmeRequest
*req
= opaque
;
477 NvmeSQueue
*sq
= req
->sq
;
478 NvmeCtrl
*n
= sq
->ctrl
;
479 NvmeCQueue
*cq
= n
->cq
[sq
->cqid
];
481 trace_pci_nvme_rw_cb(nvme_cid(req
));
484 block_acct_done(blk_get_stats(n
->conf
.blk
), &req
->acct
);
485 req
->status
= NVME_SUCCESS
;
487 block_acct_failed(blk_get_stats(n
->conf
.blk
), &req
->acct
);
488 req
->status
= NVME_INTERNAL_DEV_ERROR
;
491 qemu_sglist_destroy(&req
->qsg
);
493 nvme_enqueue_req_completion(cq
, req
);
496 static uint16_t nvme_flush(NvmeCtrl
*n
, NvmeNamespace
*ns
, NvmeCmd
*cmd
,
500 block_acct_start(blk_get_stats(n
->conf
.blk
), &req
->acct
, 0,
502 req
->aiocb
= blk_aio_flush(n
->conf
.blk
, nvme_rw_cb
, req
);
504 return NVME_NO_COMPLETE
;
507 static uint16_t nvme_write_zeros(NvmeCtrl
*n
, NvmeNamespace
*ns
, NvmeCmd
*cmd
,
510 NvmeRwCmd
*rw
= (NvmeRwCmd
*)cmd
;
511 const uint8_t lba_index
= NVME_ID_NS_FLBAS_INDEX(ns
->id_ns
.flbas
);
512 const uint8_t data_shift
= ns
->id_ns
.lbaf
[lba_index
].ds
;
513 uint64_t slba
= le64_to_cpu(rw
->slba
);
514 uint32_t nlb
= le16_to_cpu(rw
->nlb
) + 1;
515 uint64_t offset
= slba
<< data_shift
;
516 uint32_t count
= nlb
<< data_shift
;
518 trace_pci_nvme_write_zeroes(nvme_cid(req
), slba
, nlb
);
520 if (unlikely(slba
+ nlb
> ns
->id_ns
.nsze
)) {
521 trace_pci_nvme_err_invalid_lba_range(slba
, nlb
, ns
->id_ns
.nsze
);
522 return NVME_LBA_RANGE
| NVME_DNR
;
526 block_acct_start(blk_get_stats(n
->conf
.blk
), &req
->acct
, 0,
528 req
->aiocb
= blk_aio_pwrite_zeroes(n
->conf
.blk
, offset
, count
,
529 BDRV_REQ_MAY_UNMAP
, nvme_rw_cb
, req
);
530 return NVME_NO_COMPLETE
;
533 static uint16_t nvme_rw(NvmeCtrl
*n
, NvmeNamespace
*ns
, NvmeCmd
*cmd
,
536 NvmeRwCmd
*rw
= (NvmeRwCmd
*)cmd
;
537 uint32_t nlb
= le32_to_cpu(rw
->nlb
) + 1;
538 uint64_t slba
= le64_to_cpu(rw
->slba
);
539 uint64_t prp1
= le64_to_cpu(rw
->dptr
.prp1
);
540 uint64_t prp2
= le64_to_cpu(rw
->dptr
.prp2
);
542 uint8_t lba_index
= NVME_ID_NS_FLBAS_INDEX(ns
->id_ns
.flbas
);
543 uint8_t data_shift
= ns
->id_ns
.lbaf
[lba_index
].ds
;
544 uint64_t data_size
= (uint64_t)nlb
<< data_shift
;
545 uint64_t data_offset
= slba
<< data_shift
;
546 int is_write
= rw
->opcode
== NVME_CMD_WRITE
? 1 : 0;
547 enum BlockAcctType acct
= is_write
? BLOCK_ACCT_WRITE
: BLOCK_ACCT_READ
;
549 trace_pci_nvme_rw(is_write
? "write" : "read", nlb
, data_size
, slba
);
551 if (unlikely((slba
+ nlb
) > ns
->id_ns
.nsze
)) {
552 block_acct_invalid(blk_get_stats(n
->conf
.blk
), acct
);
553 trace_pci_nvme_err_invalid_lba_range(slba
, nlb
, ns
->id_ns
.nsze
);
554 return NVME_LBA_RANGE
| NVME_DNR
;
557 if (nvme_map_prp(&req
->qsg
, &req
->iov
, prp1
, prp2
, data_size
, n
)) {
558 block_acct_invalid(blk_get_stats(n
->conf
.blk
), acct
);
559 return NVME_INVALID_FIELD
| NVME_DNR
;
562 dma_acct_start(n
->conf
.blk
, &req
->acct
, &req
->qsg
, acct
);
563 if (req
->qsg
.nsg
> 0) {
565 req
->aiocb
= is_write
?
566 dma_blk_write(n
->conf
.blk
, &req
->qsg
, data_offset
, BDRV_SECTOR_SIZE
,
568 dma_blk_read(n
->conf
.blk
, &req
->qsg
, data_offset
, BDRV_SECTOR_SIZE
,
572 req
->aiocb
= is_write
?
573 blk_aio_pwritev(n
->conf
.blk
, data_offset
, &req
->iov
, 0, nvme_rw_cb
,
575 blk_aio_preadv(n
->conf
.blk
, data_offset
, &req
->iov
, 0, nvme_rw_cb
,
579 return NVME_NO_COMPLETE
;
582 static uint16_t nvme_io_cmd(NvmeCtrl
*n
, NvmeCmd
*cmd
, NvmeRequest
*req
)
585 uint32_t nsid
= le32_to_cpu(cmd
->nsid
);
587 trace_pci_nvme_io_cmd(nvme_cid(req
), nsid
, nvme_sqid(req
), cmd
->opcode
);
589 if (unlikely(nsid
== 0 || nsid
> n
->num_namespaces
)) {
590 trace_pci_nvme_err_invalid_ns(nsid
, n
->num_namespaces
);
591 return NVME_INVALID_NSID
| NVME_DNR
;
594 ns
= &n
->namespaces
[nsid
- 1];
595 switch (cmd
->opcode
) {
597 return nvme_flush(n
, ns
, cmd
, req
);
598 case NVME_CMD_WRITE_ZEROS
:
599 return nvme_write_zeros(n
, ns
, cmd
, req
);
602 return nvme_rw(n
, ns
, cmd
, req
);
604 trace_pci_nvme_err_invalid_opc(cmd
->opcode
);
605 return NVME_INVALID_OPCODE
| NVME_DNR
;
609 static void nvme_free_sq(NvmeSQueue
*sq
, NvmeCtrl
*n
)
611 n
->sq
[sq
->sqid
] = NULL
;
612 timer_del(sq
->timer
);
613 timer_free(sq
->timer
);
620 static uint16_t nvme_del_sq(NvmeCtrl
*n
, NvmeCmd
*cmd
)
622 NvmeDeleteQ
*c
= (NvmeDeleteQ
*)cmd
;
623 NvmeRequest
*req
, *next
;
626 uint16_t qid
= le16_to_cpu(c
->qid
);
628 if (unlikely(!qid
|| nvme_check_sqid(n
, qid
))) {
629 trace_pci_nvme_err_invalid_del_sq(qid
);
630 return NVME_INVALID_QID
| NVME_DNR
;
633 trace_pci_nvme_del_sq(qid
);
636 while (!QTAILQ_EMPTY(&sq
->out_req_list
)) {
637 req
= QTAILQ_FIRST(&sq
->out_req_list
);
639 blk_aio_cancel(req
->aiocb
);
641 if (!nvme_check_cqid(n
, sq
->cqid
)) {
642 cq
= n
->cq
[sq
->cqid
];
643 QTAILQ_REMOVE(&cq
->sq_list
, sq
, entry
);
646 QTAILQ_FOREACH_SAFE(req
, &cq
->req_list
, entry
, next
) {
648 QTAILQ_REMOVE(&cq
->req_list
, req
, entry
);
649 QTAILQ_INSERT_TAIL(&sq
->req_list
, req
, entry
);
658 static void nvme_init_sq(NvmeSQueue
*sq
, NvmeCtrl
*n
, uint64_t dma_addr
,
659 uint16_t sqid
, uint16_t cqid
, uint16_t size
)
665 sq
->dma_addr
= dma_addr
;
669 sq
->head
= sq
->tail
= 0;
670 sq
->io_req
= g_new(NvmeRequest
, sq
->size
);
672 QTAILQ_INIT(&sq
->req_list
);
673 QTAILQ_INIT(&sq
->out_req_list
);
674 for (i
= 0; i
< sq
->size
; i
++) {
675 sq
->io_req
[i
].sq
= sq
;
676 QTAILQ_INSERT_TAIL(&(sq
->req_list
), &sq
->io_req
[i
], entry
);
678 sq
->timer
= timer_new_ns(QEMU_CLOCK_VIRTUAL
, nvme_process_sq
, sq
);
682 QTAILQ_INSERT_TAIL(&(cq
->sq_list
), sq
, entry
);
686 static uint16_t nvme_create_sq(NvmeCtrl
*n
, NvmeCmd
*cmd
)
689 NvmeCreateSq
*c
= (NvmeCreateSq
*)cmd
;
691 uint16_t cqid
= le16_to_cpu(c
->cqid
);
692 uint16_t sqid
= le16_to_cpu(c
->sqid
);
693 uint16_t qsize
= le16_to_cpu(c
->qsize
);
694 uint16_t qflags
= le16_to_cpu(c
->sq_flags
);
695 uint64_t prp1
= le64_to_cpu(c
->prp1
);
697 trace_pci_nvme_create_sq(prp1
, sqid
, cqid
, qsize
, qflags
);
699 if (unlikely(!cqid
|| nvme_check_cqid(n
, cqid
))) {
700 trace_pci_nvme_err_invalid_create_sq_cqid(cqid
);
701 return NVME_INVALID_CQID
| NVME_DNR
;
703 if (unlikely(!sqid
|| !nvme_check_sqid(n
, sqid
))) {
704 trace_pci_nvme_err_invalid_create_sq_sqid(sqid
);
705 return NVME_INVALID_QID
| NVME_DNR
;
707 if (unlikely(!qsize
|| qsize
> NVME_CAP_MQES(n
->bar
.cap
))) {
708 trace_pci_nvme_err_invalid_create_sq_size(qsize
);
709 return NVME_MAX_QSIZE_EXCEEDED
| NVME_DNR
;
711 if (unlikely(!prp1
|| prp1
& (n
->page_size
- 1))) {
712 trace_pci_nvme_err_invalid_create_sq_addr(prp1
);
713 return NVME_INVALID_FIELD
| NVME_DNR
;
715 if (unlikely(!(NVME_SQ_FLAGS_PC(qflags
)))) {
716 trace_pci_nvme_err_invalid_create_sq_qflags(NVME_SQ_FLAGS_PC(qflags
));
717 return NVME_INVALID_FIELD
| NVME_DNR
;
719 sq
= g_malloc0(sizeof(*sq
));
720 nvme_init_sq(sq
, n
, prp1
, sqid
, cqid
, qsize
+ 1);
724 static uint16_t nvme_smart_info(NvmeCtrl
*n
, NvmeCmd
*cmd
, uint8_t rae
,
725 uint32_t buf_len
, uint64_t off
,
728 uint64_t prp1
= le64_to_cpu(cmd
->dptr
.prp1
);
729 uint64_t prp2
= le64_to_cpu(cmd
->dptr
.prp2
);
730 uint32_t nsid
= le32_to_cpu(cmd
->nsid
);
734 uint64_t units_read
= 0, units_written
= 0;
735 uint64_t read_commands
= 0, write_commands
= 0;
739 if (nsid
&& nsid
!= 0xffffffff) {
740 return NVME_INVALID_FIELD
| NVME_DNR
;
743 s
= blk_get_stats(n
->conf
.blk
);
745 units_read
= s
->nr_bytes
[BLOCK_ACCT_READ
] >> BDRV_SECTOR_BITS
;
746 units_written
= s
->nr_bytes
[BLOCK_ACCT_WRITE
] >> BDRV_SECTOR_BITS
;
747 read_commands
= s
->nr_ops
[BLOCK_ACCT_READ
];
748 write_commands
= s
->nr_ops
[BLOCK_ACCT_WRITE
];
750 if (off
> sizeof(smart
)) {
751 return NVME_INVALID_FIELD
| NVME_DNR
;
754 trans_len
= MIN(sizeof(smart
) - off
, buf_len
);
756 memset(&smart
, 0x0, sizeof(smart
));
758 smart
.data_units_read
[0] = cpu_to_le64(DIV_ROUND_UP(units_read
, 1000));
759 smart
.data_units_written
[0] = cpu_to_le64(DIV_ROUND_UP(units_written
,
761 smart
.host_read_commands
[0] = cpu_to_le64(read_commands
);
762 smart
.host_write_commands
[0] = cpu_to_le64(write_commands
);
764 smart
.temperature
= cpu_to_le16(n
->temperature
);
766 if ((n
->temperature
>= n
->features
.temp_thresh_hi
) ||
767 (n
->temperature
<= n
->features
.temp_thresh_low
)) {
768 smart
.critical_warning
|= NVME_SMART_TEMPERATURE
;
771 current_ms
= qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
);
772 smart
.power_on_hours
[0] =
773 cpu_to_le64((((current_ms
- n
->starttime_ms
) / 1000) / 60) / 60);
776 nvme_clear_events(n
, NVME_AER_TYPE_SMART
);
779 return nvme_dma_read_prp(n
, (uint8_t *) &smart
+ off
, trans_len
, prp1
,
783 static uint16_t nvme_fw_log_info(NvmeCtrl
*n
, NvmeCmd
*cmd
, uint32_t buf_len
,
784 uint64_t off
, NvmeRequest
*req
)
787 uint64_t prp1
= le64_to_cpu(cmd
->dptr
.prp1
);
788 uint64_t prp2
= le64_to_cpu(cmd
->dptr
.prp2
);
789 NvmeFwSlotInfoLog fw_log
= {
793 strpadcpy((char *)&fw_log
.frs1
, sizeof(fw_log
.frs1
), "1.0", ' ');
795 if (off
> sizeof(fw_log
)) {
796 return NVME_INVALID_FIELD
| NVME_DNR
;
799 trans_len
= MIN(sizeof(fw_log
) - off
, buf_len
);
801 return nvme_dma_read_prp(n
, (uint8_t *) &fw_log
+ off
, trans_len
, prp1
,
805 static uint16_t nvme_error_info(NvmeCtrl
*n
, NvmeCmd
*cmd
, uint8_t rae
,
806 uint32_t buf_len
, uint64_t off
,
810 uint64_t prp1
= le64_to_cpu(cmd
->dptr
.prp1
);
811 uint64_t prp2
= le64_to_cpu(cmd
->dptr
.prp2
);
815 nvme_clear_events(n
, NVME_AER_TYPE_ERROR
);
818 if (off
> sizeof(errlog
)) {
819 return NVME_INVALID_FIELD
| NVME_DNR
;
822 memset(&errlog
, 0x0, sizeof(errlog
));
824 trans_len
= MIN(sizeof(errlog
) - off
, buf_len
);
826 return nvme_dma_read_prp(n
, (uint8_t *)&errlog
, trans_len
, prp1
, prp2
);
829 static uint16_t nvme_get_log(NvmeCtrl
*n
, NvmeCmd
*cmd
, NvmeRequest
*req
)
831 uint32_t dw10
= le32_to_cpu(cmd
->cdw10
);
832 uint32_t dw11
= le32_to_cpu(cmd
->cdw11
);
833 uint32_t dw12
= le32_to_cpu(cmd
->cdw12
);
834 uint32_t dw13
= le32_to_cpu(cmd
->cdw13
);
835 uint8_t lid
= dw10
& 0xff;
836 uint8_t lsp
= (dw10
>> 8) & 0xf;
837 uint8_t rae
= (dw10
>> 15) & 0x1;
838 uint32_t numdl
, numdu
;
839 uint64_t off
, lpol
, lpou
;
842 numdl
= (dw10
>> 16);
843 numdu
= (dw11
& 0xffff);
847 len
= (((numdu
<< 16) | numdl
) + 1) << 2;
848 off
= (lpou
<< 32ULL) | lpol
;
851 return NVME_INVALID_FIELD
| NVME_DNR
;
854 trace_pci_nvme_get_log(nvme_cid(req
), lid
, lsp
, rae
, len
, off
);
857 case NVME_LOG_ERROR_INFO
:
858 return nvme_error_info(n
, cmd
, rae
, len
, off
, req
);
859 case NVME_LOG_SMART_INFO
:
860 return nvme_smart_info(n
, cmd
, rae
, len
, off
, req
);
861 case NVME_LOG_FW_SLOT_INFO
:
862 return nvme_fw_log_info(n
, cmd
, len
, off
, req
);
864 trace_pci_nvme_err_invalid_log_page(nvme_cid(req
), lid
);
865 return NVME_INVALID_FIELD
| NVME_DNR
;
869 static void nvme_free_cq(NvmeCQueue
*cq
, NvmeCtrl
*n
)
871 n
->cq
[cq
->cqid
] = NULL
;
872 timer_del(cq
->timer
);
873 timer_free(cq
->timer
);
874 msix_vector_unuse(&n
->parent_obj
, cq
->vector
);
880 static uint16_t nvme_del_cq(NvmeCtrl
*n
, NvmeCmd
*cmd
)
882 NvmeDeleteQ
*c
= (NvmeDeleteQ
*)cmd
;
884 uint16_t qid
= le16_to_cpu(c
->qid
);
886 if (unlikely(!qid
|| nvme_check_cqid(n
, qid
))) {
887 trace_pci_nvme_err_invalid_del_cq_cqid(qid
);
888 return NVME_INVALID_CQID
| NVME_DNR
;
892 if (unlikely(!QTAILQ_EMPTY(&cq
->sq_list
))) {
893 trace_pci_nvme_err_invalid_del_cq_notempty(qid
);
894 return NVME_INVALID_QUEUE_DEL
;
896 nvme_irq_deassert(n
, cq
);
897 trace_pci_nvme_del_cq(qid
);
902 static void nvme_init_cq(NvmeCQueue
*cq
, NvmeCtrl
*n
, uint64_t dma_addr
,
903 uint16_t cqid
, uint16_t vector
, uint16_t size
, uint16_t irq_enabled
)
907 ret
= msix_vector_use(&n
->parent_obj
, vector
);
912 cq
->dma_addr
= dma_addr
;
914 cq
->irq_enabled
= irq_enabled
;
916 cq
->head
= cq
->tail
= 0;
917 QTAILQ_INIT(&cq
->req_list
);
918 QTAILQ_INIT(&cq
->sq_list
);
920 cq
->timer
= timer_new_ns(QEMU_CLOCK_VIRTUAL
, nvme_post_cqes
, cq
);
923 static uint16_t nvme_create_cq(NvmeCtrl
*n
, NvmeCmd
*cmd
)
926 NvmeCreateCq
*c
= (NvmeCreateCq
*)cmd
;
927 uint16_t cqid
= le16_to_cpu(c
->cqid
);
928 uint16_t vector
= le16_to_cpu(c
->irq_vector
);
929 uint16_t qsize
= le16_to_cpu(c
->qsize
);
930 uint16_t qflags
= le16_to_cpu(c
->cq_flags
);
931 uint64_t prp1
= le64_to_cpu(c
->prp1
);
933 trace_pci_nvme_create_cq(prp1
, cqid
, vector
, qsize
, qflags
,
934 NVME_CQ_FLAGS_IEN(qflags
) != 0);
936 if (unlikely(!cqid
|| !nvme_check_cqid(n
, cqid
))) {
937 trace_pci_nvme_err_invalid_create_cq_cqid(cqid
);
938 return NVME_INVALID_CQID
| NVME_DNR
;
940 if (unlikely(!qsize
|| qsize
> NVME_CAP_MQES(n
->bar
.cap
))) {
941 trace_pci_nvme_err_invalid_create_cq_size(qsize
);
942 return NVME_MAX_QSIZE_EXCEEDED
| NVME_DNR
;
944 if (unlikely(!prp1
)) {
945 trace_pci_nvme_err_invalid_create_cq_addr(prp1
);
946 return NVME_INVALID_FIELD
| NVME_DNR
;
948 if (unlikely(!msix_enabled(&n
->parent_obj
) && vector
)) {
949 trace_pci_nvme_err_invalid_create_cq_vector(vector
);
950 return NVME_INVALID_IRQ_VECTOR
| NVME_DNR
;
952 if (unlikely(vector
>= n
->params
.msix_qsize
)) {
953 trace_pci_nvme_err_invalid_create_cq_vector(vector
);
954 return NVME_INVALID_IRQ_VECTOR
| NVME_DNR
;
956 if (unlikely(!(NVME_CQ_FLAGS_PC(qflags
)))) {
957 trace_pci_nvme_err_invalid_create_cq_qflags(NVME_CQ_FLAGS_PC(qflags
));
958 return NVME_INVALID_FIELD
| NVME_DNR
;
961 cq
= g_malloc0(sizeof(*cq
));
962 nvme_init_cq(cq
, n
, prp1
, cqid
, vector
, qsize
+ 1,
963 NVME_CQ_FLAGS_IEN(qflags
));
966 * It is only required to set qs_created when creating a completion queue;
967 * creating a submission queue without a matching completion queue will
970 n
->qs_created
= true;
974 static uint16_t nvme_identify_ctrl(NvmeCtrl
*n
, NvmeIdentify
*c
)
976 uint64_t prp1
= le64_to_cpu(c
->prp1
);
977 uint64_t prp2
= le64_to_cpu(c
->prp2
);
979 trace_pci_nvme_identify_ctrl();
981 return nvme_dma_read_prp(n
, (uint8_t *)&n
->id_ctrl
, sizeof(n
->id_ctrl
),
985 static uint16_t nvme_identify_ns(NvmeCtrl
*n
, NvmeIdentify
*c
)
988 uint32_t nsid
= le32_to_cpu(c
->nsid
);
989 uint64_t prp1
= le64_to_cpu(c
->prp1
);
990 uint64_t prp2
= le64_to_cpu(c
->prp2
);
992 trace_pci_nvme_identify_ns(nsid
);
994 if (unlikely(nsid
== 0 || nsid
> n
->num_namespaces
)) {
995 trace_pci_nvme_err_invalid_ns(nsid
, n
->num_namespaces
);
996 return NVME_INVALID_NSID
| NVME_DNR
;
999 ns
= &n
->namespaces
[nsid
- 1];
1001 return nvme_dma_read_prp(n
, (uint8_t *)&ns
->id_ns
, sizeof(ns
->id_ns
),
1005 static uint16_t nvme_identify_nslist(NvmeCtrl
*n
, NvmeIdentify
*c
)
1007 static const int data_len
= NVME_IDENTIFY_DATA_SIZE
;
1008 uint32_t min_nsid
= le32_to_cpu(c
->nsid
);
1009 uint64_t prp1
= le64_to_cpu(c
->prp1
);
1010 uint64_t prp2
= le64_to_cpu(c
->prp2
);
1015 trace_pci_nvme_identify_nslist(min_nsid
);
1018 * Both 0xffffffff (NVME_NSID_BROADCAST) and 0xfffffffe are invalid values
1019 * since the Active Namespace ID List should return namespaces with ids
1020 * *higher* than the NSID specified in the command. This is also specified
1021 * in the spec (NVM Express v1.3d, Section 5.15.4).
1023 if (min_nsid
>= NVME_NSID_BROADCAST
- 1) {
1024 return NVME_INVALID_NSID
| NVME_DNR
;
1027 list
= g_malloc0(data_len
);
1028 for (i
= 0; i
< n
->num_namespaces
; i
++) {
1032 list
[j
++] = cpu_to_le32(i
+ 1);
1033 if (j
== data_len
/ sizeof(uint32_t)) {
1037 ret
= nvme_dma_read_prp(n
, (uint8_t *)list
, data_len
, prp1
, prp2
);
1042 static uint16_t nvme_identify_ns_descr_list(NvmeCtrl
*n
, NvmeIdentify
*c
)
1044 uint32_t nsid
= le32_to_cpu(c
->nsid
);
1045 uint64_t prp1
= le64_to_cpu(c
->prp1
);
1046 uint64_t prp2
= le64_to_cpu(c
->prp2
);
1048 uint8_t list
[NVME_IDENTIFY_DATA_SIZE
];
1057 struct data
*ns_descrs
= (struct data
*)list
;
1059 trace_pci_nvme_identify_ns_descr_list(nsid
);
1061 if (unlikely(nsid
== 0 || nsid
> n
->num_namespaces
)) {
1062 trace_pci_nvme_err_invalid_ns(nsid
, n
->num_namespaces
);
1063 return NVME_INVALID_NSID
| NVME_DNR
;
1066 memset(list
, 0x0, sizeof(list
));
1069 * Because the NGUID and EUI64 fields are 0 in the Identify Namespace data
1070 * structure, a Namespace UUID (nidt = 0x3) must be reported in the
1071 * Namespace Identification Descriptor. Add a very basic Namespace UUID
1074 ns_descrs
->uuid
.hdr
.nidt
= NVME_NIDT_UUID
;
1075 ns_descrs
->uuid
.hdr
.nidl
= NVME_NIDT_UUID_LEN
;
1076 stl_be_p(&ns_descrs
->uuid
.v
, nsid
);
1078 return nvme_dma_read_prp(n
, list
, NVME_IDENTIFY_DATA_SIZE
, prp1
, prp2
);
1081 static uint16_t nvme_identify(NvmeCtrl
*n
, NvmeCmd
*cmd
)
1083 NvmeIdentify
*c
= (NvmeIdentify
*)cmd
;
1085 switch (le32_to_cpu(c
->cns
)) {
1086 case NVME_ID_CNS_NS
:
1087 return nvme_identify_ns(n
, c
);
1088 case NVME_ID_CNS_CTRL
:
1089 return nvme_identify_ctrl(n
, c
);
1090 case NVME_ID_CNS_NS_ACTIVE_LIST
:
1091 return nvme_identify_nslist(n
, c
);
1092 case NVME_ID_CNS_NS_DESCR_LIST
:
1093 return nvme_identify_ns_descr_list(n
, c
);
1095 trace_pci_nvme_err_invalid_identify_cns(le32_to_cpu(c
->cns
));
1096 return NVME_INVALID_FIELD
| NVME_DNR
;
1100 static uint16_t nvme_abort(NvmeCtrl
*n
, NvmeCmd
*cmd
, NvmeRequest
*req
)
1102 uint16_t sqid
= le32_to_cpu(cmd
->cdw10
) & 0xffff;
1104 req
->cqe
.result
= 1;
1105 if (nvme_check_sqid(n
, sqid
)) {
1106 return NVME_INVALID_FIELD
| NVME_DNR
;
1109 return NVME_SUCCESS
;
1112 static inline void nvme_set_timestamp(NvmeCtrl
*n
, uint64_t ts
)
1114 trace_pci_nvme_setfeat_timestamp(ts
);
1116 n
->host_timestamp
= le64_to_cpu(ts
);
1117 n
->timestamp_set_qemu_clock_ms
= qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
);
1120 static inline uint64_t nvme_get_timestamp(const NvmeCtrl
*n
)
1122 uint64_t current_time
= qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
);
1123 uint64_t elapsed_time
= current_time
- n
->timestamp_set_qemu_clock_ms
;
1125 union nvme_timestamp
{
1127 uint64_t timestamp
:48;
1135 union nvme_timestamp ts
;
1139 * If the sum of the Timestamp value set by the host and the elapsed
1140 * time exceeds 2^48, the value returned should be reduced modulo 2^48.
1142 ts
.timestamp
= (n
->host_timestamp
+ elapsed_time
) & 0xffffffffffff;
1144 /* If the host timestamp is non-zero, set the timestamp origin */
1145 ts
.origin
= n
->host_timestamp
? 0x01 : 0x00;
1147 trace_pci_nvme_getfeat_timestamp(ts
.all
);
1149 return cpu_to_le64(ts
.all
);
1152 static uint16_t nvme_get_feature_timestamp(NvmeCtrl
*n
, NvmeCmd
*cmd
)
1154 uint64_t prp1
= le64_to_cpu(cmd
->dptr
.prp1
);
1155 uint64_t prp2
= le64_to_cpu(cmd
->dptr
.prp2
);
1157 uint64_t timestamp
= nvme_get_timestamp(n
);
1159 return nvme_dma_read_prp(n
, (uint8_t *)×tamp
,
1160 sizeof(timestamp
), prp1
, prp2
);
1163 static uint16_t nvme_get_feature(NvmeCtrl
*n
, NvmeCmd
*cmd
, NvmeRequest
*req
)
1165 uint32_t dw10
= le32_to_cpu(cmd
->cdw10
);
1166 uint32_t dw11
= le32_to_cpu(cmd
->cdw11
);
1167 uint32_t nsid
= le32_to_cpu(cmd
->nsid
);
1169 uint8_t fid
= NVME_GETSETFEAT_FID(dw10
);
1170 NvmeGetFeatureSelect sel
= NVME_GETFEAT_SELECT(dw10
);
1173 static const uint32_t nvme_feature_default
[NVME_FID_MAX
] = {
1174 [NVME_ARBITRATION
] = NVME_ARB_AB_NOLIMIT
,
1177 trace_pci_nvme_getfeat(nvme_cid(req
), fid
, sel
, dw11
);
1179 if (!nvme_feature_support
[fid
]) {
1180 return NVME_INVALID_FIELD
| NVME_DNR
;
1183 if (nvme_feature_cap
[fid
] & NVME_FEAT_CAP_NS
) {
1184 if (!nsid
|| nsid
> n
->num_namespaces
) {
1186 * The Reservation Notification Mask and Reservation Persistence
1187 * features require a status code of Invalid Field in Command when
1188 * NSID is 0xFFFFFFFF. Since the device does not support those
1189 * features we can always return Invalid Namespace or Format as we
1190 * should do for all other features.
1192 return NVME_INVALID_NSID
| NVME_DNR
;
1197 case NVME_GETFEAT_SELECT_CURRENT
:
1199 case NVME_GETFEAT_SELECT_SAVED
:
1200 /* no features are saveable by the controller; fallthrough */
1201 case NVME_GETFEAT_SELECT_DEFAULT
:
1203 case NVME_GETFEAT_SELECT_CAP
:
1204 result
= nvme_feature_cap
[fid
];
1209 case NVME_TEMPERATURE_THRESHOLD
:
1213 * The controller only implements the Composite Temperature sensor, so
1214 * return 0 for all other sensors.
1216 if (NVME_TEMP_TMPSEL(dw11
) != NVME_TEMP_TMPSEL_COMPOSITE
) {
1220 switch (NVME_TEMP_THSEL(dw11
)) {
1221 case NVME_TEMP_THSEL_OVER
:
1222 result
= n
->features
.temp_thresh_hi
;
1224 case NVME_TEMP_THSEL_UNDER
:
1225 result
= n
->features
.temp_thresh_low
;
1229 return NVME_INVALID_FIELD
| NVME_DNR
;
1230 case NVME_VOLATILE_WRITE_CACHE
:
1231 result
= blk_enable_write_cache(n
->conf
.blk
);
1232 trace_pci_nvme_getfeat_vwcache(result
? "enabled" : "disabled");
1234 case NVME_ASYNCHRONOUS_EVENT_CONF
:
1235 result
= n
->features
.async_config
;
1237 case NVME_TIMESTAMP
:
1238 return nvme_get_feature_timestamp(n
, cmd
);
1245 case NVME_TEMPERATURE_THRESHOLD
:
1248 if (NVME_TEMP_TMPSEL(dw11
) != NVME_TEMP_TMPSEL_COMPOSITE
) {
1252 if (NVME_TEMP_THSEL(dw11
) == NVME_TEMP_THSEL_OVER
) {
1253 result
= NVME_TEMPERATURE_WARNING
;
1257 case NVME_NUMBER_OF_QUEUES
:
1258 result
= (n
->params
.max_ioqpairs
- 1) |
1259 ((n
->params
.max_ioqpairs
- 1) << 16);
1260 trace_pci_nvme_getfeat_numq(result
);
1262 case NVME_INTERRUPT_VECTOR_CONF
:
1264 if (iv
>= n
->params
.max_ioqpairs
+ 1) {
1265 return NVME_INVALID_FIELD
| NVME_DNR
;
1269 if (iv
== n
->admin_cq
.vector
) {
1270 result
|= NVME_INTVC_NOCOALESCING
;
1275 result
= nvme_feature_default
[fid
];
1280 req
->cqe
.result
= cpu_to_le32(result
);
1281 return NVME_SUCCESS
;
1284 static uint16_t nvme_set_feature_timestamp(NvmeCtrl
*n
, NvmeCmd
*cmd
)
1288 uint64_t prp1
= le64_to_cpu(cmd
->dptr
.prp1
);
1289 uint64_t prp2
= le64_to_cpu(cmd
->dptr
.prp2
);
1291 ret
= nvme_dma_write_prp(n
, (uint8_t *)×tamp
,
1292 sizeof(timestamp
), prp1
, prp2
);
1293 if (ret
!= NVME_SUCCESS
) {
1297 nvme_set_timestamp(n
, timestamp
);
1299 return NVME_SUCCESS
;
1302 static uint16_t nvme_set_feature(NvmeCtrl
*n
, NvmeCmd
*cmd
, NvmeRequest
*req
)
1304 uint32_t dw10
= le32_to_cpu(cmd
->cdw10
);
1305 uint32_t dw11
= le32_to_cpu(cmd
->cdw11
);
1306 uint32_t nsid
= le32_to_cpu(cmd
->nsid
);
1307 uint8_t fid
= NVME_GETSETFEAT_FID(dw10
);
1308 uint8_t save
= NVME_SETFEAT_SAVE(dw10
);
1310 trace_pci_nvme_setfeat(nvme_cid(req
), fid
, save
, dw11
);
1313 return NVME_FID_NOT_SAVEABLE
| NVME_DNR
;
1316 if (!nvme_feature_support
[fid
]) {
1317 return NVME_INVALID_FIELD
| NVME_DNR
;
1320 if (nvme_feature_cap
[fid
] & NVME_FEAT_CAP_NS
) {
1321 if (!nsid
|| (nsid
!= NVME_NSID_BROADCAST
&&
1322 nsid
> n
->num_namespaces
)) {
1323 return NVME_INVALID_NSID
| NVME_DNR
;
1325 } else if (nsid
&& nsid
!= NVME_NSID_BROADCAST
) {
1326 if (nsid
> n
->num_namespaces
) {
1327 return NVME_INVALID_NSID
| NVME_DNR
;
1330 return NVME_FEAT_NOT_NS_SPEC
| NVME_DNR
;
1333 if (!(nvme_feature_cap
[fid
] & NVME_FEAT_CAP_CHANGE
)) {
1334 return NVME_FEAT_NOT_CHANGEABLE
| NVME_DNR
;
1338 case NVME_TEMPERATURE_THRESHOLD
:
1339 if (NVME_TEMP_TMPSEL(dw11
) != NVME_TEMP_TMPSEL_COMPOSITE
) {
1343 switch (NVME_TEMP_THSEL(dw11
)) {
1344 case NVME_TEMP_THSEL_OVER
:
1345 n
->features
.temp_thresh_hi
= NVME_TEMP_TMPTH(dw11
);
1347 case NVME_TEMP_THSEL_UNDER
:
1348 n
->features
.temp_thresh_low
= NVME_TEMP_TMPTH(dw11
);
1351 return NVME_INVALID_FIELD
| NVME_DNR
;
1354 if (((n
->temperature
>= n
->features
.temp_thresh_hi
) ||
1355 (n
->temperature
<= n
->features
.temp_thresh_low
)) &&
1356 NVME_AEC_SMART(n
->features
.async_config
) & NVME_SMART_TEMPERATURE
) {
1357 nvme_enqueue_event(n
, NVME_AER_TYPE_SMART
,
1358 NVME_AER_INFO_SMART_TEMP_THRESH
,
1359 NVME_LOG_SMART_INFO
);
1363 case NVME_VOLATILE_WRITE_CACHE
:
1364 if (!(dw11
& 0x1) && blk_enable_write_cache(n
->conf
.blk
)) {
1365 blk_flush(n
->conf
.blk
);
1368 blk_set_enable_write_cache(n
->conf
.blk
, dw11
& 1);
1370 case NVME_NUMBER_OF_QUEUES
:
1371 if (n
->qs_created
) {
1372 return NVME_CMD_SEQ_ERROR
| NVME_DNR
;
1376 * NVMe v1.3, Section 5.21.1.7: 0xffff is not an allowed value for NCQR
1379 if ((dw11
& 0xffff) == 0xffff || ((dw11
>> 16) & 0xffff) == 0xffff) {
1380 return NVME_INVALID_FIELD
| NVME_DNR
;
1383 trace_pci_nvme_setfeat_numq((dw11
& 0xFFFF) + 1,
1384 ((dw11
>> 16) & 0xFFFF) + 1,
1385 n
->params
.max_ioqpairs
,
1386 n
->params
.max_ioqpairs
);
1387 req
->cqe
.result
= cpu_to_le32((n
->params
.max_ioqpairs
- 1) |
1388 ((n
->params
.max_ioqpairs
- 1) << 16));
1390 case NVME_ASYNCHRONOUS_EVENT_CONF
:
1391 n
->features
.async_config
= dw11
;
1393 case NVME_TIMESTAMP
:
1394 return nvme_set_feature_timestamp(n
, cmd
);
1396 return NVME_FEAT_NOT_CHANGEABLE
| NVME_DNR
;
1398 return NVME_SUCCESS
;
1401 static uint16_t nvme_aer(NvmeCtrl
*n
, NvmeCmd
*cmd
, NvmeRequest
*req
)
1403 trace_pci_nvme_aer(nvme_cid(req
));
1405 if (n
->outstanding_aers
> n
->params
.aerl
) {
1406 trace_pci_nvme_aer_aerl_exceeded();
1407 return NVME_AER_LIMIT_EXCEEDED
;
1410 n
->aer_reqs
[n
->outstanding_aers
] = req
;
1411 n
->outstanding_aers
++;
1413 if (!QTAILQ_EMPTY(&n
->aer_queue
)) {
1414 nvme_process_aers(n
);
1417 return NVME_NO_COMPLETE
;
1420 static uint16_t nvme_admin_cmd(NvmeCtrl
*n
, NvmeCmd
*cmd
, NvmeRequest
*req
)
1422 trace_pci_nvme_admin_cmd(nvme_cid(req
), nvme_sqid(req
), cmd
->opcode
);
1424 switch (cmd
->opcode
) {
1425 case NVME_ADM_CMD_DELETE_SQ
:
1426 return nvme_del_sq(n
, cmd
);
1427 case NVME_ADM_CMD_CREATE_SQ
:
1428 return nvme_create_sq(n
, cmd
);
1429 case NVME_ADM_CMD_GET_LOG_PAGE
:
1430 return nvme_get_log(n
, cmd
, req
);
1431 case NVME_ADM_CMD_DELETE_CQ
:
1432 return nvme_del_cq(n
, cmd
);
1433 case NVME_ADM_CMD_CREATE_CQ
:
1434 return nvme_create_cq(n
, cmd
);
1435 case NVME_ADM_CMD_IDENTIFY
:
1436 return nvme_identify(n
, cmd
);
1437 case NVME_ADM_CMD_ABORT
:
1438 return nvme_abort(n
, cmd
, req
);
1439 case NVME_ADM_CMD_SET_FEATURES
:
1440 return nvme_set_feature(n
, cmd
, req
);
1441 case NVME_ADM_CMD_GET_FEATURES
:
1442 return nvme_get_feature(n
, cmd
, req
);
1443 case NVME_ADM_CMD_ASYNC_EV_REQ
:
1444 return nvme_aer(n
, cmd
, req
);
1446 trace_pci_nvme_err_invalid_admin_opc(cmd
->opcode
);
1447 return NVME_INVALID_OPCODE
| NVME_DNR
;
1451 static void nvme_process_sq(void *opaque
)
1453 NvmeSQueue
*sq
= opaque
;
1454 NvmeCtrl
*n
= sq
->ctrl
;
1455 NvmeCQueue
*cq
= n
->cq
[sq
->cqid
];
1462 while (!(nvme_sq_empty(sq
) || QTAILQ_EMPTY(&sq
->req_list
))) {
1463 addr
= sq
->dma_addr
+ sq
->head
* n
->sqe_size
;
1464 nvme_addr_read(n
, addr
, (void *)&cmd
, sizeof(cmd
));
1465 nvme_inc_sq_head(sq
);
1467 req
= QTAILQ_FIRST(&sq
->req_list
);
1468 QTAILQ_REMOVE(&sq
->req_list
, req
, entry
);
1469 QTAILQ_INSERT_TAIL(&sq
->out_req_list
, req
, entry
);
1470 memset(&req
->cqe
, 0, sizeof(req
->cqe
));
1471 req
->cqe
.cid
= cmd
.cid
;
1473 status
= sq
->sqid
? nvme_io_cmd(n
, &cmd
, req
) :
1474 nvme_admin_cmd(n
, &cmd
, req
);
1475 if (status
!= NVME_NO_COMPLETE
) {
1476 req
->status
= status
;
1477 nvme_enqueue_req_completion(cq
, req
);
1482 static void nvme_clear_ctrl(NvmeCtrl
*n
)
1486 blk_drain(n
->conf
.blk
);
1488 for (i
= 0; i
< n
->params
.max_ioqpairs
+ 1; i
++) {
1489 if (n
->sq
[i
] != NULL
) {
1490 nvme_free_sq(n
->sq
[i
], n
);
1493 for (i
= 0; i
< n
->params
.max_ioqpairs
+ 1; i
++) {
1494 if (n
->cq
[i
] != NULL
) {
1495 nvme_free_cq(n
->cq
[i
], n
);
1499 while (!QTAILQ_EMPTY(&n
->aer_queue
)) {
1500 NvmeAsyncEvent
*event
= QTAILQ_FIRST(&n
->aer_queue
);
1501 QTAILQ_REMOVE(&n
->aer_queue
, event
, entry
);
1506 n
->outstanding_aers
= 0;
1507 n
->qs_created
= false;
1509 blk_flush(n
->conf
.blk
);
1513 static int nvme_start_ctrl(NvmeCtrl
*n
)
1515 uint32_t page_bits
= NVME_CC_MPS(n
->bar
.cc
) + 12;
1516 uint32_t page_size
= 1 << page_bits
;
1518 if (unlikely(n
->cq
[0])) {
1519 trace_pci_nvme_err_startfail_cq();
1522 if (unlikely(n
->sq
[0])) {
1523 trace_pci_nvme_err_startfail_sq();
1526 if (unlikely(!n
->bar
.asq
)) {
1527 trace_pci_nvme_err_startfail_nbarasq();
1530 if (unlikely(!n
->bar
.acq
)) {
1531 trace_pci_nvme_err_startfail_nbaracq();
1534 if (unlikely(n
->bar
.asq
& (page_size
- 1))) {
1535 trace_pci_nvme_err_startfail_asq_misaligned(n
->bar
.asq
);
1538 if (unlikely(n
->bar
.acq
& (page_size
- 1))) {
1539 trace_pci_nvme_err_startfail_acq_misaligned(n
->bar
.acq
);
1542 if (unlikely(NVME_CC_MPS(n
->bar
.cc
) <
1543 NVME_CAP_MPSMIN(n
->bar
.cap
))) {
1544 trace_pci_nvme_err_startfail_page_too_small(
1545 NVME_CC_MPS(n
->bar
.cc
),
1546 NVME_CAP_MPSMIN(n
->bar
.cap
));
1549 if (unlikely(NVME_CC_MPS(n
->bar
.cc
) >
1550 NVME_CAP_MPSMAX(n
->bar
.cap
))) {
1551 trace_pci_nvme_err_startfail_page_too_large(
1552 NVME_CC_MPS(n
->bar
.cc
),
1553 NVME_CAP_MPSMAX(n
->bar
.cap
));
1556 if (unlikely(NVME_CC_IOCQES(n
->bar
.cc
) <
1557 NVME_CTRL_CQES_MIN(n
->id_ctrl
.cqes
))) {
1558 trace_pci_nvme_err_startfail_cqent_too_small(
1559 NVME_CC_IOCQES(n
->bar
.cc
),
1560 NVME_CTRL_CQES_MIN(n
->bar
.cap
));
1563 if (unlikely(NVME_CC_IOCQES(n
->bar
.cc
) >
1564 NVME_CTRL_CQES_MAX(n
->id_ctrl
.cqes
))) {
1565 trace_pci_nvme_err_startfail_cqent_too_large(
1566 NVME_CC_IOCQES(n
->bar
.cc
),
1567 NVME_CTRL_CQES_MAX(n
->bar
.cap
));
1570 if (unlikely(NVME_CC_IOSQES(n
->bar
.cc
) <
1571 NVME_CTRL_SQES_MIN(n
->id_ctrl
.sqes
))) {
1572 trace_pci_nvme_err_startfail_sqent_too_small(
1573 NVME_CC_IOSQES(n
->bar
.cc
),
1574 NVME_CTRL_SQES_MIN(n
->bar
.cap
));
1577 if (unlikely(NVME_CC_IOSQES(n
->bar
.cc
) >
1578 NVME_CTRL_SQES_MAX(n
->id_ctrl
.sqes
))) {
1579 trace_pci_nvme_err_startfail_sqent_too_large(
1580 NVME_CC_IOSQES(n
->bar
.cc
),
1581 NVME_CTRL_SQES_MAX(n
->bar
.cap
));
1584 if (unlikely(!NVME_AQA_ASQS(n
->bar
.aqa
))) {
1585 trace_pci_nvme_err_startfail_asqent_sz_zero();
1588 if (unlikely(!NVME_AQA_ACQS(n
->bar
.aqa
))) {
1589 trace_pci_nvme_err_startfail_acqent_sz_zero();
1593 n
->page_bits
= page_bits
;
1594 n
->page_size
= page_size
;
1595 n
->max_prp_ents
= n
->page_size
/ sizeof(uint64_t);
1596 n
->cqe_size
= 1 << NVME_CC_IOCQES(n
->bar
.cc
);
1597 n
->sqe_size
= 1 << NVME_CC_IOSQES(n
->bar
.cc
);
1598 nvme_init_cq(&n
->admin_cq
, n
, n
->bar
.acq
, 0, 0,
1599 NVME_AQA_ACQS(n
->bar
.aqa
) + 1, 1);
1600 nvme_init_sq(&n
->admin_sq
, n
, n
->bar
.asq
, 0, 0,
1601 NVME_AQA_ASQS(n
->bar
.aqa
) + 1);
1603 nvme_set_timestamp(n
, 0ULL);
1605 QTAILQ_INIT(&n
->aer_queue
);
1610 static void nvme_write_bar(NvmeCtrl
*n
, hwaddr offset
, uint64_t data
,
1613 if (unlikely(offset
& (sizeof(uint32_t) - 1))) {
1614 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_misaligned32
,
1615 "MMIO write not 32-bit aligned,"
1616 " offset=0x%"PRIx64
"", offset
);
1617 /* should be ignored, fall through for now */
1620 if (unlikely(size
< sizeof(uint32_t))) {
1621 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_toosmall
,
1622 "MMIO write smaller than 32-bits,"
1623 " offset=0x%"PRIx64
", size=%u",
1625 /* should be ignored, fall through for now */
1629 case 0xc: /* INTMS */
1630 if (unlikely(msix_enabled(&(n
->parent_obj
)))) {
1631 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix
,
1632 "undefined access to interrupt mask set"
1633 " when MSI-X is enabled");
1634 /* should be ignored, fall through for now */
1636 n
->bar
.intms
|= data
& 0xffffffff;
1637 n
->bar
.intmc
= n
->bar
.intms
;
1638 trace_pci_nvme_mmio_intm_set(data
& 0xffffffff, n
->bar
.intmc
);
1641 case 0x10: /* INTMC */
1642 if (unlikely(msix_enabled(&(n
->parent_obj
)))) {
1643 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix
,
1644 "undefined access to interrupt mask clr"
1645 " when MSI-X is enabled");
1646 /* should be ignored, fall through for now */
1648 n
->bar
.intms
&= ~(data
& 0xffffffff);
1649 n
->bar
.intmc
= n
->bar
.intms
;
1650 trace_pci_nvme_mmio_intm_clr(data
& 0xffffffff, n
->bar
.intmc
);
1654 trace_pci_nvme_mmio_cfg(data
& 0xffffffff);
1655 /* Windows first sends data, then sends enable bit */
1656 if (!NVME_CC_EN(data
) && !NVME_CC_EN(n
->bar
.cc
) &&
1657 !NVME_CC_SHN(data
) && !NVME_CC_SHN(n
->bar
.cc
))
1662 if (NVME_CC_EN(data
) && !NVME_CC_EN(n
->bar
.cc
)) {
1664 if (unlikely(nvme_start_ctrl(n
))) {
1665 trace_pci_nvme_err_startfail();
1666 n
->bar
.csts
= NVME_CSTS_FAILED
;
1668 trace_pci_nvme_mmio_start_success();
1669 n
->bar
.csts
= NVME_CSTS_READY
;
1671 } else if (!NVME_CC_EN(data
) && NVME_CC_EN(n
->bar
.cc
)) {
1672 trace_pci_nvme_mmio_stopped();
1674 n
->bar
.csts
&= ~NVME_CSTS_READY
;
1676 if (NVME_CC_SHN(data
) && !(NVME_CC_SHN(n
->bar
.cc
))) {
1677 trace_pci_nvme_mmio_shutdown_set();
1680 n
->bar
.csts
|= NVME_CSTS_SHST_COMPLETE
;
1681 } else if (!NVME_CC_SHN(data
) && NVME_CC_SHN(n
->bar
.cc
)) {
1682 trace_pci_nvme_mmio_shutdown_cleared();
1683 n
->bar
.csts
&= ~NVME_CSTS_SHST_COMPLETE
;
1687 case 0x1C: /* CSTS */
1688 if (data
& (1 << 4)) {
1689 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ssreset_w1c_unsupported
,
1690 "attempted to W1C CSTS.NSSRO"
1691 " but CAP.NSSRS is zero (not supported)");
1692 } else if (data
!= 0) {
1693 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ro_csts
,
1694 "attempted to set a read only bit"
1695 " of controller status");
1698 case 0x20: /* NSSR */
1699 if (data
== 0x4E564D65) {
1700 trace_pci_nvme_ub_mmiowr_ssreset_unsupported();
1702 /* The spec says that writes of other values have no effect */
1706 case 0x24: /* AQA */
1707 n
->bar
.aqa
= data
& 0xffffffff;
1708 trace_pci_nvme_mmio_aqattr(data
& 0xffffffff);
1710 case 0x28: /* ASQ */
1712 trace_pci_nvme_mmio_asqaddr(data
);
1714 case 0x2c: /* ASQ hi */
1715 n
->bar
.asq
|= data
<< 32;
1716 trace_pci_nvme_mmio_asqaddr_hi(data
, n
->bar
.asq
);
1718 case 0x30: /* ACQ */
1719 trace_pci_nvme_mmio_acqaddr(data
);
1722 case 0x34: /* ACQ hi */
1723 n
->bar
.acq
|= data
<< 32;
1724 trace_pci_nvme_mmio_acqaddr_hi(data
, n
->bar
.acq
);
1726 case 0x38: /* CMBLOC */
1727 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbloc_reserved
,
1728 "invalid write to reserved CMBLOC"
1729 " when CMBSZ is zero, ignored");
1731 case 0x3C: /* CMBSZ */
1732 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbsz_readonly
,
1733 "invalid write to read only CMBSZ, ignored");
1735 case 0xE00: /* PMRCAP */
1736 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrcap_readonly
,
1737 "invalid write to PMRCAP register, ignored");
1739 case 0xE04: /* TODO PMRCTL */
1741 case 0xE08: /* PMRSTS */
1742 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrsts_readonly
,
1743 "invalid write to PMRSTS register, ignored");
1745 case 0xE0C: /* PMREBS */
1746 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrebs_readonly
,
1747 "invalid write to PMREBS register, ignored");
1749 case 0xE10: /* PMRSWTP */
1750 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrswtp_readonly
,
1751 "invalid write to PMRSWTP register, ignored");
1753 case 0xE14: /* TODO PMRMSC */
1756 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_invalid
,
1757 "invalid MMIO write,"
1758 " offset=0x%"PRIx64
", data=%"PRIx64
"",
1764 static uint64_t nvme_mmio_read(void *opaque
, hwaddr addr
, unsigned size
)
1766 NvmeCtrl
*n
= (NvmeCtrl
*)opaque
;
1767 uint8_t *ptr
= (uint8_t *)&n
->bar
;
1770 trace_pci_nvme_mmio_read(addr
);
1772 if (unlikely(addr
& (sizeof(uint32_t) - 1))) {
1773 NVME_GUEST_ERR(pci_nvme_ub_mmiord_misaligned32
,
1774 "MMIO read not 32-bit aligned,"
1775 " offset=0x%"PRIx64
"", addr
);
1776 /* should RAZ, fall through for now */
1777 } else if (unlikely(size
< sizeof(uint32_t))) {
1778 NVME_GUEST_ERR(pci_nvme_ub_mmiord_toosmall
,
1779 "MMIO read smaller than 32-bits,"
1780 " offset=0x%"PRIx64
"", addr
);
1781 /* should RAZ, fall through for now */
1784 if (addr
< sizeof(n
->bar
)) {
1786 * When PMRWBM bit 1 is set then read from
1787 * from PMRSTS should ensure prior writes
1788 * made it to persistent media
1790 if (addr
== 0xE08 &&
1791 (NVME_PMRCAP_PMRWBM(n
->bar
.pmrcap
) & 0x02)) {
1792 memory_region_msync(&n
->pmrdev
->mr
, 0, n
->pmrdev
->size
);
1794 memcpy(&val
, ptr
+ addr
, size
);
1796 NVME_GUEST_ERR(pci_nvme_ub_mmiord_invalid_ofs
,
1797 "MMIO read beyond last register,"
1798 " offset=0x%"PRIx64
", returning 0", addr
);
1804 static void nvme_process_db(NvmeCtrl
*n
, hwaddr addr
, int val
)
1808 if (unlikely(addr
& ((1 << 2) - 1))) {
1809 NVME_GUEST_ERR(pci_nvme_ub_db_wr_misaligned
,
1810 "doorbell write not 32-bit aligned,"
1811 " offset=0x%"PRIx64
", ignoring", addr
);
1815 if (((addr
- 0x1000) >> 2) & 1) {
1816 /* Completion queue doorbell write */
1818 uint16_t new_head
= val
& 0xffff;
1822 qid
= (addr
- (0x1000 + (1 << 2))) >> 3;
1823 if (unlikely(nvme_check_cqid(n
, qid
))) {
1824 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cq
,
1825 "completion queue doorbell write"
1826 " for nonexistent queue,"
1827 " sqid=%"PRIu32
", ignoring", qid
);
1830 * NVM Express v1.3d, Section 4.1 state: "If host software writes
1831 * an invalid value to the Submission Queue Tail Doorbell or
1832 * Completion Queue Head Doorbell regiter and an Asynchronous Event
1833 * Request command is outstanding, then an asynchronous event is
1834 * posted to the Admin Completion Queue with a status code of
1835 * Invalid Doorbell Write Value."
1837 * Also note that the spec includes the "Invalid Doorbell Register"
1838 * status code, but nowhere does it specify when to use it.
1839 * However, it seems reasonable to use it here in a similar
1842 if (n
->outstanding_aers
) {
1843 nvme_enqueue_event(n
, NVME_AER_TYPE_ERROR
,
1844 NVME_AER_INFO_ERR_INVALID_DB_REGISTER
,
1845 NVME_LOG_ERROR_INFO
);
1852 if (unlikely(new_head
>= cq
->size
)) {
1853 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cqhead
,
1854 "completion queue doorbell write value"
1855 " beyond queue size, sqid=%"PRIu32
","
1856 " new_head=%"PRIu16
", ignoring",
1859 if (n
->outstanding_aers
) {
1860 nvme_enqueue_event(n
, NVME_AER_TYPE_ERROR
,
1861 NVME_AER_INFO_ERR_INVALID_DB_VALUE
,
1862 NVME_LOG_ERROR_INFO
);
1868 trace_pci_nvme_mmio_doorbell_cq(cq
->cqid
, new_head
);
1870 start_sqs
= nvme_cq_full(cq
) ? 1 : 0;
1871 cq
->head
= new_head
;
1874 QTAILQ_FOREACH(sq
, &cq
->sq_list
, entry
) {
1875 timer_mod(sq
->timer
, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL
) + 500);
1877 timer_mod(cq
->timer
, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL
) + 500);
1880 if (cq
->tail
== cq
->head
) {
1881 nvme_irq_deassert(n
, cq
);
1884 /* Submission queue doorbell write */
1886 uint16_t new_tail
= val
& 0xffff;
1889 qid
= (addr
- 0x1000) >> 3;
1890 if (unlikely(nvme_check_sqid(n
, qid
))) {
1891 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sq
,
1892 "submission queue doorbell write"
1893 " for nonexistent queue,"
1894 " sqid=%"PRIu32
", ignoring", qid
);
1896 if (n
->outstanding_aers
) {
1897 nvme_enqueue_event(n
, NVME_AER_TYPE_ERROR
,
1898 NVME_AER_INFO_ERR_INVALID_DB_REGISTER
,
1899 NVME_LOG_ERROR_INFO
);
1906 if (unlikely(new_tail
>= sq
->size
)) {
1907 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sqtail
,
1908 "submission queue doorbell write value"
1909 " beyond queue size, sqid=%"PRIu32
","
1910 " new_tail=%"PRIu16
", ignoring",
1913 if (n
->outstanding_aers
) {
1914 nvme_enqueue_event(n
, NVME_AER_TYPE_ERROR
,
1915 NVME_AER_INFO_ERR_INVALID_DB_VALUE
,
1916 NVME_LOG_ERROR_INFO
);
1922 trace_pci_nvme_mmio_doorbell_sq(sq
->sqid
, new_tail
);
1924 sq
->tail
= new_tail
;
1925 timer_mod(sq
->timer
, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL
) + 500);
1929 static void nvme_mmio_write(void *opaque
, hwaddr addr
, uint64_t data
,
1932 NvmeCtrl
*n
= (NvmeCtrl
*)opaque
;
1934 trace_pci_nvme_mmio_write(addr
, data
);
1936 if (addr
< sizeof(n
->bar
)) {
1937 nvme_write_bar(n
, addr
, data
, size
);
1939 nvme_process_db(n
, addr
, data
);
1943 static const MemoryRegionOps nvme_mmio_ops
= {
1944 .read
= nvme_mmio_read
,
1945 .write
= nvme_mmio_write
,
1946 .endianness
= DEVICE_LITTLE_ENDIAN
,
1948 .min_access_size
= 2,
1949 .max_access_size
= 8,
1953 static void nvme_cmb_write(void *opaque
, hwaddr addr
, uint64_t data
,
1956 NvmeCtrl
*n
= (NvmeCtrl
*)opaque
;
1957 stn_le_p(&n
->cmbuf
[addr
], size
, data
);
1960 static uint64_t nvme_cmb_read(void *opaque
, hwaddr addr
, unsigned size
)
1962 NvmeCtrl
*n
= (NvmeCtrl
*)opaque
;
1963 return ldn_le_p(&n
->cmbuf
[addr
], size
);
1966 static const MemoryRegionOps nvme_cmb_ops
= {
1967 .read
= nvme_cmb_read
,
1968 .write
= nvme_cmb_write
,
1969 .endianness
= DEVICE_LITTLE_ENDIAN
,
1971 .min_access_size
= 1,
1972 .max_access_size
= 8,
1976 static void nvme_check_constraints(NvmeCtrl
*n
, Error
**errp
)
1978 NvmeParams
*params
= &n
->params
;
1980 if (params
->num_queues
) {
1981 warn_report("num_queues is deprecated; please use max_ioqpairs "
1984 params
->max_ioqpairs
= params
->num_queues
- 1;
1987 if (params
->max_ioqpairs
< 1 ||
1988 params
->max_ioqpairs
> NVME_MAX_IOQPAIRS
) {
1989 error_setg(errp
, "max_ioqpairs must be between 1 and %d",
1994 if (params
->msix_qsize
< 1 ||
1995 params
->msix_qsize
> PCI_MSIX_FLAGS_QSIZE
+ 1) {
1996 error_setg(errp
, "msix_qsize must be between 1 and %d",
1997 PCI_MSIX_FLAGS_QSIZE
+ 1);
2002 error_setg(errp
, "drive property not set");
2006 if (!params
->serial
) {
2007 error_setg(errp
, "serial property not set");
2011 if (!n
->params
.cmb_size_mb
&& n
->pmrdev
) {
2012 if (host_memory_backend_is_mapped(n
->pmrdev
)) {
2013 error_setg(errp
, "can't use already busy memdev: %s",
2014 object_get_canonical_path_component(OBJECT(n
->pmrdev
)));
2018 if (!is_power_of_2(n
->pmrdev
->size
)) {
2019 error_setg(errp
, "pmr backend size needs to be power of 2 in size");
2023 host_memory_backend_set_mapped(n
->pmrdev
, true);
2027 static void nvme_init_state(NvmeCtrl
*n
)
2029 n
->num_namespaces
= 1;
2030 /* add one to max_ioqpairs to account for the admin queue pair */
2031 n
->reg_size
= pow2ceil(sizeof(NvmeBar
) +
2032 2 * (n
->params
.max_ioqpairs
+ 1) * NVME_DB_SIZE
);
2033 n
->namespaces
= g_new0(NvmeNamespace
, n
->num_namespaces
);
2034 n
->sq
= g_new0(NvmeSQueue
*, n
->params
.max_ioqpairs
+ 1);
2035 n
->cq
= g_new0(NvmeCQueue
*, n
->params
.max_ioqpairs
+ 1);
2036 n
->temperature
= NVME_TEMPERATURE
;
2037 n
->features
.temp_thresh_hi
= NVME_TEMPERATURE_WARNING
;
2038 n
->starttime_ms
= qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
);
2039 n
->aer_reqs
= g_new0(NvmeRequest
*, n
->params
.aerl
+ 1);
2042 static void nvme_init_blk(NvmeCtrl
*n
, Error
**errp
)
2044 if (!blkconf_blocksizes(&n
->conf
, errp
)) {
2047 blkconf_apply_backend_options(&n
->conf
, blk_is_read_only(n
->conf
.blk
),
2051 static void nvme_init_namespace(NvmeCtrl
*n
, NvmeNamespace
*ns
, Error
**errp
)
2054 NvmeIdNs
*id_ns
= &ns
->id_ns
;
2056 bs_size
= blk_getlength(n
->conf
.blk
);
2058 error_setg_errno(errp
, -bs_size
, "could not get backing file size");
2062 n
->ns_size
= bs_size
;
2064 id_ns
->lbaf
[0].ds
= BDRV_SECTOR_BITS
;
2065 id_ns
->nsze
= cpu_to_le64(nvme_ns_nlbas(n
, ns
));
2067 /* no thin provisioning */
2068 id_ns
->ncap
= id_ns
->nsze
;
2069 id_ns
->nuse
= id_ns
->ncap
;
2072 static void nvme_init_cmb(NvmeCtrl
*n
, PCIDevice
*pci_dev
)
2074 NVME_CMBLOC_SET_BIR(n
->bar
.cmbloc
, NVME_CMB_BIR
);
2075 NVME_CMBLOC_SET_OFST(n
->bar
.cmbloc
, 0);
2077 NVME_CMBSZ_SET_SQS(n
->bar
.cmbsz
, 1);
2078 NVME_CMBSZ_SET_CQS(n
->bar
.cmbsz
, 0);
2079 NVME_CMBSZ_SET_LISTS(n
->bar
.cmbsz
, 0);
2080 NVME_CMBSZ_SET_RDS(n
->bar
.cmbsz
, 1);
2081 NVME_CMBSZ_SET_WDS(n
->bar
.cmbsz
, 1);
2082 NVME_CMBSZ_SET_SZU(n
->bar
.cmbsz
, 2); /* MBs */
2083 NVME_CMBSZ_SET_SZ(n
->bar
.cmbsz
, n
->params
.cmb_size_mb
);
2085 n
->cmbuf
= g_malloc0(NVME_CMBSZ_GETSIZE(n
->bar
.cmbsz
));
2086 memory_region_init_io(&n
->ctrl_mem
, OBJECT(n
), &nvme_cmb_ops
, n
,
2087 "nvme-cmb", NVME_CMBSZ_GETSIZE(n
->bar
.cmbsz
));
2088 pci_register_bar(pci_dev
, NVME_CMBLOC_BIR(n
->bar
.cmbloc
),
2089 PCI_BASE_ADDRESS_SPACE_MEMORY
|
2090 PCI_BASE_ADDRESS_MEM_TYPE_64
|
2091 PCI_BASE_ADDRESS_MEM_PREFETCH
, &n
->ctrl_mem
);
2094 static void nvme_init_pmr(NvmeCtrl
*n
, PCIDevice
*pci_dev
)
2096 /* Controller Capabilities register */
2097 NVME_CAP_SET_PMRS(n
->bar
.cap
, 1);
2099 /* PMR Capabities register */
2101 NVME_PMRCAP_SET_RDS(n
->bar
.pmrcap
, 0);
2102 NVME_PMRCAP_SET_WDS(n
->bar
.pmrcap
, 0);
2103 NVME_PMRCAP_SET_BIR(n
->bar
.pmrcap
, NVME_PMR_BIR
);
2104 NVME_PMRCAP_SET_PMRTU(n
->bar
.pmrcap
, 0);
2105 /* Turn on bit 1 support */
2106 NVME_PMRCAP_SET_PMRWBM(n
->bar
.pmrcap
, 0x02);
2107 NVME_PMRCAP_SET_PMRTO(n
->bar
.pmrcap
, 0);
2108 NVME_PMRCAP_SET_CMSS(n
->bar
.pmrcap
, 0);
2110 /* PMR Control register */
2112 NVME_PMRCTL_SET_EN(n
->bar
.pmrctl
, 0);
2114 /* PMR Status register */
2116 NVME_PMRSTS_SET_ERR(n
->bar
.pmrsts
, 0);
2117 NVME_PMRSTS_SET_NRDY(n
->bar
.pmrsts
, 0);
2118 NVME_PMRSTS_SET_HSTS(n
->bar
.pmrsts
, 0);
2119 NVME_PMRSTS_SET_CBAI(n
->bar
.pmrsts
, 0);
2121 /* PMR Elasticity Buffer Size register */
2123 NVME_PMREBS_SET_PMRSZU(n
->bar
.pmrebs
, 0);
2124 NVME_PMREBS_SET_RBB(n
->bar
.pmrebs
, 0);
2125 NVME_PMREBS_SET_PMRWBZ(n
->bar
.pmrebs
, 0);
2127 /* PMR Sustained Write Throughput register */
2129 NVME_PMRSWTP_SET_PMRSWTU(n
->bar
.pmrswtp
, 0);
2130 NVME_PMRSWTP_SET_PMRSWTV(n
->bar
.pmrswtp
, 0);
2132 /* PMR Memory Space Control register */
2134 NVME_PMRMSC_SET_CMSE(n
->bar
.pmrmsc
, 0);
2135 NVME_PMRMSC_SET_CBA(n
->bar
.pmrmsc
, 0);
2137 pci_register_bar(pci_dev
, NVME_PMRCAP_BIR(n
->bar
.pmrcap
),
2138 PCI_BASE_ADDRESS_SPACE_MEMORY
|
2139 PCI_BASE_ADDRESS_MEM_TYPE_64
|
2140 PCI_BASE_ADDRESS_MEM_PREFETCH
, &n
->pmrdev
->mr
);
2143 static void nvme_init_pci(NvmeCtrl
*n
, PCIDevice
*pci_dev
, Error
**errp
)
2145 uint8_t *pci_conf
= pci_dev
->config
;
2147 pci_conf
[PCI_INTERRUPT_PIN
] = 1;
2148 pci_config_set_prog_interface(pci_conf
, 0x2);
2149 pci_config_set_class(pci_conf
, PCI_CLASS_STORAGE_EXPRESS
);
2150 pcie_endpoint_cap_init(pci_dev
, 0x80);
2152 memory_region_init_io(&n
->iomem
, OBJECT(n
), &nvme_mmio_ops
, n
, "nvme",
2154 pci_register_bar(pci_dev
, 0, PCI_BASE_ADDRESS_SPACE_MEMORY
|
2155 PCI_BASE_ADDRESS_MEM_TYPE_64
, &n
->iomem
);
2156 if (msix_init_exclusive_bar(pci_dev
, n
->params
.msix_qsize
, 4, errp
)) {
2160 if (n
->params
.cmb_size_mb
) {
2161 nvme_init_cmb(n
, pci_dev
);
2162 } else if (n
->pmrdev
) {
2163 nvme_init_pmr(n
, pci_dev
);
2167 static void nvme_init_ctrl(NvmeCtrl
*n
, PCIDevice
*pci_dev
)
2169 NvmeIdCtrl
*id
= &n
->id_ctrl
;
2170 uint8_t *pci_conf
= pci_dev
->config
;
2172 id
->vid
= cpu_to_le16(pci_get_word(pci_conf
+ PCI_VENDOR_ID
));
2173 id
->ssvid
= cpu_to_le16(pci_get_word(pci_conf
+ PCI_SUBSYSTEM_VENDOR_ID
));
2174 strpadcpy((char *)id
->mn
, sizeof(id
->mn
), "QEMU NVMe Ctrl", ' ');
2175 strpadcpy((char *)id
->fr
, sizeof(id
->fr
), "1.0", ' ');
2176 strpadcpy((char *)id
->sn
, sizeof(id
->sn
), n
->params
.serial
, ' ');
2181 id
->oacs
= cpu_to_le16(0);
2184 * Because the controller always completes the Abort command immediately,
2185 * there can never be more than one concurrently executing Abort command,
2186 * so this value is never used for anything. Note that there can easily be
2187 * many Abort commands in the queues, but they are not considered
2188 * "executing" until processed by nvme_abort.
2190 * The specification recommends a value of 3 for Abort Command Limit (four
2191 * concurrently outstanding Abort commands), so lets use that though it is
2195 id
->aerl
= n
->params
.aerl
;
2196 id
->frmw
= (NVME_NUM_FW_SLOTS
<< 1) | NVME_FRMW_SLOT1_RO
;
2197 id
->lpa
= NVME_LPA_EXTENDED
;
2199 /* recommended default value (~70 C) */
2200 id
->wctemp
= cpu_to_le16(NVME_TEMPERATURE_WARNING
);
2201 id
->cctemp
= cpu_to_le16(NVME_TEMPERATURE_CRITICAL
);
2203 id
->sqes
= (0x6 << 4) | 0x6;
2204 id
->cqes
= (0x4 << 4) | 0x4;
2205 id
->nn
= cpu_to_le32(n
->num_namespaces
);
2206 id
->oncs
= cpu_to_le16(NVME_ONCS_WRITE_ZEROS
| NVME_ONCS_TIMESTAMP
|
2207 NVME_ONCS_FEATURES
);
2209 id
->psd
[0].mp
= cpu_to_le16(0x9c4);
2210 id
->psd
[0].enlat
= cpu_to_le32(0x10);
2211 id
->psd
[0].exlat
= cpu_to_le32(0x4);
2212 if (blk_enable_write_cache(n
->conf
.blk
)) {
2217 NVME_CAP_SET_MQES(n
->bar
.cap
, 0x7ff);
2218 NVME_CAP_SET_CQR(n
->bar
.cap
, 1);
2219 NVME_CAP_SET_TO(n
->bar
.cap
, 0xf);
2220 NVME_CAP_SET_CSS(n
->bar
.cap
, 1);
2221 NVME_CAP_SET_MPSMAX(n
->bar
.cap
, 4);
2223 n
->bar
.vs
= 0x00010200;
2224 n
->bar
.intmc
= n
->bar
.intms
= 0;
2227 static void nvme_realize(PCIDevice
*pci_dev
, Error
**errp
)
2229 NvmeCtrl
*n
= NVME(pci_dev
);
2230 Error
*local_err
= NULL
;
2234 nvme_check_constraints(n
, &local_err
);
2236 error_propagate(errp
, local_err
);
2241 nvme_init_blk(n
, &local_err
);
2243 error_propagate(errp
, local_err
);
2247 nvme_init_pci(n
, pci_dev
, &local_err
);
2249 error_propagate(errp
, local_err
);
2253 nvme_init_ctrl(n
, pci_dev
);
2255 for (i
= 0; i
< n
->num_namespaces
; i
++) {
2256 nvme_init_namespace(n
, &n
->namespaces
[i
], &local_err
);
2258 error_propagate(errp
, local_err
);
2264 static void nvme_exit(PCIDevice
*pci_dev
)
2266 NvmeCtrl
*n
= NVME(pci_dev
);
2269 g_free(n
->namespaces
);
2272 g_free(n
->aer_reqs
);
2274 if (n
->params
.cmb_size_mb
) {
2279 host_memory_backend_set_mapped(n
->pmrdev
, false);
2281 msix_uninit_exclusive_bar(pci_dev
);
2284 static Property nvme_props
[] = {
2285 DEFINE_BLOCK_PROPERTIES(NvmeCtrl
, conf
),
2286 DEFINE_PROP_LINK("pmrdev", NvmeCtrl
, pmrdev
, TYPE_MEMORY_BACKEND
,
2287 HostMemoryBackend
*),
2288 DEFINE_PROP_STRING("serial", NvmeCtrl
, params
.serial
),
2289 DEFINE_PROP_UINT32("cmb_size_mb", NvmeCtrl
, params
.cmb_size_mb
, 0),
2290 DEFINE_PROP_UINT32("num_queues", NvmeCtrl
, params
.num_queues
, 0),
2291 DEFINE_PROP_UINT32("max_ioqpairs", NvmeCtrl
, params
.max_ioqpairs
, 64),
2292 DEFINE_PROP_UINT16("msix_qsize", NvmeCtrl
, params
.msix_qsize
, 65),
2293 DEFINE_PROP_UINT8("aerl", NvmeCtrl
, params
.aerl
, 3),
2294 DEFINE_PROP_UINT32("aer_max_queued", NvmeCtrl
, params
.aer_max_queued
, 64),
2295 DEFINE_PROP_END_OF_LIST(),
2298 static const VMStateDescription nvme_vmstate
= {
2303 static void nvme_class_init(ObjectClass
*oc
, void *data
)
2305 DeviceClass
*dc
= DEVICE_CLASS(oc
);
2306 PCIDeviceClass
*pc
= PCI_DEVICE_CLASS(oc
);
2308 pc
->realize
= nvme_realize
;
2309 pc
->exit
= nvme_exit
;
2310 pc
->class_id
= PCI_CLASS_STORAGE_EXPRESS
;
2311 pc
->vendor_id
= PCI_VENDOR_ID_INTEL
;
2312 pc
->device_id
= 0x5845;
2315 set_bit(DEVICE_CATEGORY_STORAGE
, dc
->categories
);
2316 dc
->desc
= "Non-Volatile Memory Express";
2317 device_class_set_props(dc
, nvme_props
);
2318 dc
->vmsd
= &nvme_vmstate
;
2321 static void nvme_instance_init(Object
*obj
)
2323 NvmeCtrl
*s
= NVME(obj
);
2325 device_add_bootindex_property(obj
, &s
->conf
.bootindex
,
2326 "bootindex", "/namespace@1,0",
2330 static const TypeInfo nvme_info
= {
2332 .parent
= TYPE_PCI_DEVICE
,
2333 .instance_size
= sizeof(NvmeCtrl
),
2334 .class_init
= nvme_class_init
,
2335 .instance_init
= nvme_instance_init
,
2336 .interfaces
= (InterfaceInfo
[]) {
2337 { INTERFACE_PCIE_DEVICE
},
2342 static void nvme_register_types(void)
2344 type_register_static(&nvme_info
);
2347 type_init(nvme_register_types
)