2 * QEMU NVM Express Controller
4 * Copyright (c) 2012, Intel Corporation
6 * Written by Keith Busch <keith.busch@intel.com>
8 * This code is licensed under the GNU GPL v2 or later.
12 * Reference Specs: http://www.nvmexpress.org, 1.2, 1.1, 1.0e
14 * http://www.nvmexpress.org/resources/
19 * -drive file=<file>,if=none,id=<drive_id>
20 * -device nvme,drive=<drive_id>,serial=<serial>,id=<id[optional]>, \
21 * cmb_size_mb=<cmb_size_mb[optional]>, \
22 * [pmrdev=<mem_backend_file_id>,] \
23 * max_ioqpairs=<N[optional]>
25 * Note cmb_size_mb denotes size of CMB in MB. CMB is assumed to be at
26 * offset 0 in BAR2 and supports only WDS, RDS and SQS for now.
28 * cmb_size_mb= and pmrdev= options are mutually exclusive due to limitation
29 * in available BAR's. cmb_size_mb= will take precedence over pmrdev= when
31 * Enabling pmr emulation can be achieved by pointing to memory-backend-file.
33 * -object memory-backend-file,id=<mem_id>,share=on,mem-path=<file_path>, \
34 * size=<size> .... -device nvme,...,pmrdev=<mem_id>
37 #include "qemu/osdep.h"
38 #include "qemu/units.h"
39 #include "qemu/error-report.h"
40 #include "hw/block/block.h"
41 #include "hw/pci/msix.h"
42 #include "hw/pci/pci.h"
43 #include "hw/qdev-properties.h"
44 #include "migration/vmstate.h"
45 #include "sysemu/sysemu.h"
46 #include "qapi/error.h"
47 #include "qapi/visitor.h"
48 #include "sysemu/hostmem.h"
49 #include "sysemu/block-backend.h"
50 #include "exec/memory.h"
52 #include "qemu/module.h"
53 #include "qemu/cutils.h"
57 #define NVME_REG_SIZE 0x1000
58 #define NVME_DB_SIZE 4
59 #define NVME_CMB_BIR 2
60 #define NVME_PMR_BIR 2
62 #define NVME_GUEST_ERR(trace, fmt, ...) \
64 (trace_##trace)(__VA_ARGS__); \
65 qemu_log_mask(LOG_GUEST_ERROR, #trace \
66 " in %s: " fmt "\n", __func__, ## __VA_ARGS__); \
69 static void nvme_process_sq(void *opaque
);
71 static bool nvme_addr_is_cmb(NvmeCtrl
*n
, hwaddr addr
)
73 hwaddr low
= n
->ctrl_mem
.addr
;
74 hwaddr hi
= n
->ctrl_mem
.addr
+ int128_get64(n
->ctrl_mem
.size
);
76 return addr
>= low
&& addr
< hi
;
79 static void nvme_addr_read(NvmeCtrl
*n
, hwaddr addr
, void *buf
, int size
)
81 if (n
->bar
.cmbsz
&& nvme_addr_is_cmb(n
, addr
)) {
82 memcpy(buf
, (void *)&n
->cmbuf
[addr
- n
->ctrl_mem
.addr
], size
);
86 pci_dma_read(&n
->parent_obj
, addr
, buf
, size
);
89 static int nvme_check_sqid(NvmeCtrl
*n
, uint16_t sqid
)
91 return sqid
< n
->params
.max_ioqpairs
+ 1 && n
->sq
[sqid
] != NULL
? 0 : -1;
94 static int nvme_check_cqid(NvmeCtrl
*n
, uint16_t cqid
)
96 return cqid
< n
->params
.max_ioqpairs
+ 1 && n
->cq
[cqid
] != NULL
? 0 : -1;
99 static void nvme_inc_cq_tail(NvmeCQueue
*cq
)
102 if (cq
->tail
>= cq
->size
) {
104 cq
->phase
= !cq
->phase
;
108 static void nvme_inc_sq_head(NvmeSQueue
*sq
)
110 sq
->head
= (sq
->head
+ 1) % sq
->size
;
113 static uint8_t nvme_cq_full(NvmeCQueue
*cq
)
115 return (cq
->tail
+ 1) % cq
->size
== cq
->head
;
118 static uint8_t nvme_sq_empty(NvmeSQueue
*sq
)
120 return sq
->head
== sq
->tail
;
123 static void nvme_irq_check(NvmeCtrl
*n
)
125 if (msix_enabled(&(n
->parent_obj
))) {
128 if (~n
->bar
.intms
& n
->irq_status
) {
129 pci_irq_assert(&n
->parent_obj
);
131 pci_irq_deassert(&n
->parent_obj
);
135 static void nvme_irq_assert(NvmeCtrl
*n
, NvmeCQueue
*cq
)
137 if (cq
->irq_enabled
) {
138 if (msix_enabled(&(n
->parent_obj
))) {
139 trace_pci_nvme_irq_msix(cq
->vector
);
140 msix_notify(&(n
->parent_obj
), cq
->vector
);
142 trace_pci_nvme_irq_pin();
143 assert(cq
->vector
< 32);
144 n
->irq_status
|= 1 << cq
->vector
;
148 trace_pci_nvme_irq_masked();
152 static void nvme_irq_deassert(NvmeCtrl
*n
, NvmeCQueue
*cq
)
154 if (cq
->irq_enabled
) {
155 if (msix_enabled(&(n
->parent_obj
))) {
158 assert(cq
->vector
< 32);
159 n
->irq_status
&= ~(1 << cq
->vector
);
165 static uint16_t nvme_map_prp(QEMUSGList
*qsg
, QEMUIOVector
*iov
, uint64_t prp1
,
166 uint64_t prp2
, uint32_t len
, NvmeCtrl
*n
)
168 hwaddr trans_len
= n
->page_size
- (prp1
% n
->page_size
);
169 trans_len
= MIN(len
, trans_len
);
170 int num_prps
= (len
>> n
->page_bits
) + 1;
172 if (unlikely(!prp1
)) {
173 trace_pci_nvme_err_invalid_prp();
174 return NVME_INVALID_FIELD
| NVME_DNR
;
175 } else if (n
->bar
.cmbsz
&& prp1
>= n
->ctrl_mem
.addr
&&
176 prp1
< n
->ctrl_mem
.addr
+ int128_get64(n
->ctrl_mem
.size
)) {
178 qemu_iovec_init(iov
, num_prps
);
179 qemu_iovec_add(iov
, (void *)&n
->cmbuf
[prp1
- n
->ctrl_mem
.addr
], trans_len
);
181 pci_dma_sglist_init(qsg
, &n
->parent_obj
, num_prps
);
182 qemu_sglist_add(qsg
, prp1
, trans_len
);
186 if (unlikely(!prp2
)) {
187 trace_pci_nvme_err_invalid_prp2_missing();
190 if (len
> n
->page_size
) {
191 uint64_t prp_list
[n
->max_prp_ents
];
192 uint32_t nents
, prp_trans
;
195 nents
= (len
+ n
->page_size
- 1) >> n
->page_bits
;
196 prp_trans
= MIN(n
->max_prp_ents
, nents
) * sizeof(uint64_t);
197 nvme_addr_read(n
, prp2
, (void *)prp_list
, prp_trans
);
199 uint64_t prp_ent
= le64_to_cpu(prp_list
[i
]);
201 if (i
== n
->max_prp_ents
- 1 && len
> n
->page_size
) {
202 if (unlikely(!prp_ent
|| prp_ent
& (n
->page_size
- 1))) {
203 trace_pci_nvme_err_invalid_prplist_ent(prp_ent
);
208 nents
= (len
+ n
->page_size
- 1) >> n
->page_bits
;
209 prp_trans
= MIN(n
->max_prp_ents
, nents
) * sizeof(uint64_t);
210 nvme_addr_read(n
, prp_ent
, (void *)prp_list
,
212 prp_ent
= le64_to_cpu(prp_list
[i
]);
215 if (unlikely(!prp_ent
|| prp_ent
& (n
->page_size
- 1))) {
216 trace_pci_nvme_err_invalid_prplist_ent(prp_ent
);
220 trans_len
= MIN(len
, n
->page_size
);
222 qemu_sglist_add(qsg
, prp_ent
, trans_len
);
224 qemu_iovec_add(iov
, (void *)&n
->cmbuf
[prp_ent
- n
->ctrl_mem
.addr
], trans_len
);
230 if (unlikely(prp2
& (n
->page_size
- 1))) {
231 trace_pci_nvme_err_invalid_prp2_align(prp2
);
235 qemu_sglist_add(qsg
, prp2
, len
);
237 qemu_iovec_add(iov
, (void *)&n
->cmbuf
[prp2
- n
->ctrl_mem
.addr
], trans_len
);
244 qemu_sglist_destroy(qsg
);
245 return NVME_INVALID_FIELD
| NVME_DNR
;
248 static uint16_t nvme_dma_write_prp(NvmeCtrl
*n
, uint8_t *ptr
, uint32_t len
,
249 uint64_t prp1
, uint64_t prp2
)
253 uint16_t status
= NVME_SUCCESS
;
255 if (nvme_map_prp(&qsg
, &iov
, prp1
, prp2
, len
, n
)) {
256 return NVME_INVALID_FIELD
| NVME_DNR
;
259 if (dma_buf_write(ptr
, len
, &qsg
)) {
260 status
= NVME_INVALID_FIELD
| NVME_DNR
;
262 qemu_sglist_destroy(&qsg
);
264 if (qemu_iovec_to_buf(&iov
, 0, ptr
, len
) != len
) {
265 status
= NVME_INVALID_FIELD
| NVME_DNR
;
267 qemu_iovec_destroy(&iov
);
272 static uint16_t nvme_dma_read_prp(NvmeCtrl
*n
, uint8_t *ptr
, uint32_t len
,
273 uint64_t prp1
, uint64_t prp2
)
277 uint16_t status
= NVME_SUCCESS
;
279 trace_pci_nvme_dma_read(prp1
, prp2
);
281 if (nvme_map_prp(&qsg
, &iov
, prp1
, prp2
, len
, n
)) {
282 return NVME_INVALID_FIELD
| NVME_DNR
;
285 if (unlikely(dma_buf_read(ptr
, len
, &qsg
))) {
286 trace_pci_nvme_err_invalid_dma();
287 status
= NVME_INVALID_FIELD
| NVME_DNR
;
289 qemu_sglist_destroy(&qsg
);
291 if (unlikely(qemu_iovec_from_buf(&iov
, 0, ptr
, len
) != len
)) {
292 trace_pci_nvme_err_invalid_dma();
293 status
= NVME_INVALID_FIELD
| NVME_DNR
;
295 qemu_iovec_destroy(&iov
);
300 static void nvme_post_cqes(void *opaque
)
302 NvmeCQueue
*cq
= opaque
;
303 NvmeCtrl
*n
= cq
->ctrl
;
304 NvmeRequest
*req
, *next
;
306 QTAILQ_FOREACH_SAFE(req
, &cq
->req_list
, entry
, next
) {
310 if (nvme_cq_full(cq
)) {
314 QTAILQ_REMOVE(&cq
->req_list
, req
, entry
);
316 req
->cqe
.status
= cpu_to_le16((req
->status
<< 1) | cq
->phase
);
317 req
->cqe
.sq_id
= cpu_to_le16(sq
->sqid
);
318 req
->cqe
.sq_head
= cpu_to_le16(sq
->head
);
319 addr
= cq
->dma_addr
+ cq
->tail
* n
->cqe_size
;
320 nvme_inc_cq_tail(cq
);
321 pci_dma_write(&n
->parent_obj
, addr
, (void *)&req
->cqe
,
323 QTAILQ_INSERT_TAIL(&sq
->req_list
, req
, entry
);
325 if (cq
->tail
!= cq
->head
) {
326 nvme_irq_assert(n
, cq
);
330 static void nvme_enqueue_req_completion(NvmeCQueue
*cq
, NvmeRequest
*req
)
332 assert(cq
->cqid
== req
->sq
->cqid
);
333 QTAILQ_REMOVE(&req
->sq
->out_req_list
, req
, entry
);
334 QTAILQ_INSERT_TAIL(&cq
->req_list
, req
, entry
);
335 timer_mod(cq
->timer
, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL
) + 500);
338 static void nvme_rw_cb(void *opaque
, int ret
)
340 NvmeRequest
*req
= opaque
;
341 NvmeSQueue
*sq
= req
->sq
;
342 NvmeCtrl
*n
= sq
->ctrl
;
343 NvmeCQueue
*cq
= n
->cq
[sq
->cqid
];
346 block_acct_done(blk_get_stats(n
->conf
.blk
), &req
->acct
);
347 req
->status
= NVME_SUCCESS
;
349 block_acct_failed(blk_get_stats(n
->conf
.blk
), &req
->acct
);
350 req
->status
= NVME_INTERNAL_DEV_ERROR
;
353 qemu_sglist_destroy(&req
->qsg
);
355 nvme_enqueue_req_completion(cq
, req
);
358 static uint16_t nvme_flush(NvmeCtrl
*n
, NvmeNamespace
*ns
, NvmeCmd
*cmd
,
362 block_acct_start(blk_get_stats(n
->conf
.blk
), &req
->acct
, 0,
364 req
->aiocb
= blk_aio_flush(n
->conf
.blk
, nvme_rw_cb
, req
);
366 return NVME_NO_COMPLETE
;
369 static uint16_t nvme_write_zeros(NvmeCtrl
*n
, NvmeNamespace
*ns
, NvmeCmd
*cmd
,
372 NvmeRwCmd
*rw
= (NvmeRwCmd
*)cmd
;
373 const uint8_t lba_index
= NVME_ID_NS_FLBAS_INDEX(ns
->id_ns
.flbas
);
374 const uint8_t data_shift
= ns
->id_ns
.lbaf
[lba_index
].ds
;
375 uint64_t slba
= le64_to_cpu(rw
->slba
);
376 uint32_t nlb
= le16_to_cpu(rw
->nlb
) + 1;
377 uint64_t offset
= slba
<< data_shift
;
378 uint32_t count
= nlb
<< data_shift
;
380 if (unlikely(slba
+ nlb
> ns
->id_ns
.nsze
)) {
381 trace_pci_nvme_err_invalid_lba_range(slba
, nlb
, ns
->id_ns
.nsze
);
382 return NVME_LBA_RANGE
| NVME_DNR
;
386 block_acct_start(blk_get_stats(n
->conf
.blk
), &req
->acct
, 0,
388 req
->aiocb
= blk_aio_pwrite_zeroes(n
->conf
.blk
, offset
, count
,
389 BDRV_REQ_MAY_UNMAP
, nvme_rw_cb
, req
);
390 return NVME_NO_COMPLETE
;
393 static uint16_t nvme_rw(NvmeCtrl
*n
, NvmeNamespace
*ns
, NvmeCmd
*cmd
,
396 NvmeRwCmd
*rw
= (NvmeRwCmd
*)cmd
;
397 uint32_t nlb
= le32_to_cpu(rw
->nlb
) + 1;
398 uint64_t slba
= le64_to_cpu(rw
->slba
);
399 uint64_t prp1
= le64_to_cpu(rw
->prp1
);
400 uint64_t prp2
= le64_to_cpu(rw
->prp2
);
402 uint8_t lba_index
= NVME_ID_NS_FLBAS_INDEX(ns
->id_ns
.flbas
);
403 uint8_t data_shift
= ns
->id_ns
.lbaf
[lba_index
].ds
;
404 uint64_t data_size
= (uint64_t)nlb
<< data_shift
;
405 uint64_t data_offset
= slba
<< data_shift
;
406 int is_write
= rw
->opcode
== NVME_CMD_WRITE
? 1 : 0;
407 enum BlockAcctType acct
= is_write
? BLOCK_ACCT_WRITE
: BLOCK_ACCT_READ
;
409 trace_pci_nvme_rw(is_write
? "write" : "read", nlb
, data_size
, slba
);
411 if (unlikely((slba
+ nlb
) > ns
->id_ns
.nsze
)) {
412 block_acct_invalid(blk_get_stats(n
->conf
.blk
), acct
);
413 trace_pci_nvme_err_invalid_lba_range(slba
, nlb
, ns
->id_ns
.nsze
);
414 return NVME_LBA_RANGE
| NVME_DNR
;
417 if (nvme_map_prp(&req
->qsg
, &req
->iov
, prp1
, prp2
, data_size
, n
)) {
418 block_acct_invalid(blk_get_stats(n
->conf
.blk
), acct
);
419 return NVME_INVALID_FIELD
| NVME_DNR
;
422 dma_acct_start(n
->conf
.blk
, &req
->acct
, &req
->qsg
, acct
);
423 if (req
->qsg
.nsg
> 0) {
425 req
->aiocb
= is_write
?
426 dma_blk_write(n
->conf
.blk
, &req
->qsg
, data_offset
, BDRV_SECTOR_SIZE
,
428 dma_blk_read(n
->conf
.blk
, &req
->qsg
, data_offset
, BDRV_SECTOR_SIZE
,
432 req
->aiocb
= is_write
?
433 blk_aio_pwritev(n
->conf
.blk
, data_offset
, &req
->iov
, 0, nvme_rw_cb
,
435 blk_aio_preadv(n
->conf
.blk
, data_offset
, &req
->iov
, 0, nvme_rw_cb
,
439 return NVME_NO_COMPLETE
;
442 static uint16_t nvme_io_cmd(NvmeCtrl
*n
, NvmeCmd
*cmd
, NvmeRequest
*req
)
445 uint32_t nsid
= le32_to_cpu(cmd
->nsid
);
447 if (unlikely(nsid
== 0 || nsid
> n
->num_namespaces
)) {
448 trace_pci_nvme_err_invalid_ns(nsid
, n
->num_namespaces
);
449 return NVME_INVALID_NSID
| NVME_DNR
;
452 ns
= &n
->namespaces
[nsid
- 1];
453 switch (cmd
->opcode
) {
455 return nvme_flush(n
, ns
, cmd
, req
);
456 case NVME_CMD_WRITE_ZEROS
:
457 return nvme_write_zeros(n
, ns
, cmd
, req
);
460 return nvme_rw(n
, ns
, cmd
, req
);
462 trace_pci_nvme_err_invalid_opc(cmd
->opcode
);
463 return NVME_INVALID_OPCODE
| NVME_DNR
;
467 static void nvme_free_sq(NvmeSQueue
*sq
, NvmeCtrl
*n
)
469 n
->sq
[sq
->sqid
] = NULL
;
470 timer_del(sq
->timer
);
471 timer_free(sq
->timer
);
478 static uint16_t nvme_del_sq(NvmeCtrl
*n
, NvmeCmd
*cmd
)
480 NvmeDeleteQ
*c
= (NvmeDeleteQ
*)cmd
;
481 NvmeRequest
*req
, *next
;
484 uint16_t qid
= le16_to_cpu(c
->qid
);
486 if (unlikely(!qid
|| nvme_check_sqid(n
, qid
))) {
487 trace_pci_nvme_err_invalid_del_sq(qid
);
488 return NVME_INVALID_QID
| NVME_DNR
;
491 trace_pci_nvme_del_sq(qid
);
494 while (!QTAILQ_EMPTY(&sq
->out_req_list
)) {
495 req
= QTAILQ_FIRST(&sq
->out_req_list
);
497 blk_aio_cancel(req
->aiocb
);
499 if (!nvme_check_cqid(n
, sq
->cqid
)) {
500 cq
= n
->cq
[sq
->cqid
];
501 QTAILQ_REMOVE(&cq
->sq_list
, sq
, entry
);
504 QTAILQ_FOREACH_SAFE(req
, &cq
->req_list
, entry
, next
) {
506 QTAILQ_REMOVE(&cq
->req_list
, req
, entry
);
507 QTAILQ_INSERT_TAIL(&sq
->req_list
, req
, entry
);
516 static void nvme_init_sq(NvmeSQueue
*sq
, NvmeCtrl
*n
, uint64_t dma_addr
,
517 uint16_t sqid
, uint16_t cqid
, uint16_t size
)
523 sq
->dma_addr
= dma_addr
;
527 sq
->head
= sq
->tail
= 0;
528 sq
->io_req
= g_new(NvmeRequest
, sq
->size
);
530 QTAILQ_INIT(&sq
->req_list
);
531 QTAILQ_INIT(&sq
->out_req_list
);
532 for (i
= 0; i
< sq
->size
; i
++) {
533 sq
->io_req
[i
].sq
= sq
;
534 QTAILQ_INSERT_TAIL(&(sq
->req_list
), &sq
->io_req
[i
], entry
);
536 sq
->timer
= timer_new_ns(QEMU_CLOCK_VIRTUAL
, nvme_process_sq
, sq
);
540 QTAILQ_INSERT_TAIL(&(cq
->sq_list
), sq
, entry
);
544 static uint16_t nvme_create_sq(NvmeCtrl
*n
, NvmeCmd
*cmd
)
547 NvmeCreateSq
*c
= (NvmeCreateSq
*)cmd
;
549 uint16_t cqid
= le16_to_cpu(c
->cqid
);
550 uint16_t sqid
= le16_to_cpu(c
->sqid
);
551 uint16_t qsize
= le16_to_cpu(c
->qsize
);
552 uint16_t qflags
= le16_to_cpu(c
->sq_flags
);
553 uint64_t prp1
= le64_to_cpu(c
->prp1
);
555 trace_pci_nvme_create_sq(prp1
, sqid
, cqid
, qsize
, qflags
);
557 if (unlikely(!cqid
|| nvme_check_cqid(n
, cqid
))) {
558 trace_pci_nvme_err_invalid_create_sq_cqid(cqid
);
559 return NVME_INVALID_CQID
| NVME_DNR
;
561 if (unlikely(!sqid
|| !nvme_check_sqid(n
, sqid
))) {
562 trace_pci_nvme_err_invalid_create_sq_sqid(sqid
);
563 return NVME_INVALID_QID
| NVME_DNR
;
565 if (unlikely(!qsize
|| qsize
> NVME_CAP_MQES(n
->bar
.cap
))) {
566 trace_pci_nvme_err_invalid_create_sq_size(qsize
);
567 return NVME_MAX_QSIZE_EXCEEDED
| NVME_DNR
;
569 if (unlikely(!prp1
|| prp1
& (n
->page_size
- 1))) {
570 trace_pci_nvme_err_invalid_create_sq_addr(prp1
);
571 return NVME_INVALID_FIELD
| NVME_DNR
;
573 if (unlikely(!(NVME_SQ_FLAGS_PC(qflags
)))) {
574 trace_pci_nvme_err_invalid_create_sq_qflags(NVME_SQ_FLAGS_PC(qflags
));
575 return NVME_INVALID_FIELD
| NVME_DNR
;
577 sq
= g_malloc0(sizeof(*sq
));
578 nvme_init_sq(sq
, n
, prp1
, sqid
, cqid
, qsize
+ 1);
582 static void nvme_free_cq(NvmeCQueue
*cq
, NvmeCtrl
*n
)
584 n
->cq
[cq
->cqid
] = NULL
;
585 timer_del(cq
->timer
);
586 timer_free(cq
->timer
);
587 msix_vector_unuse(&n
->parent_obj
, cq
->vector
);
593 static uint16_t nvme_del_cq(NvmeCtrl
*n
, NvmeCmd
*cmd
)
595 NvmeDeleteQ
*c
= (NvmeDeleteQ
*)cmd
;
597 uint16_t qid
= le16_to_cpu(c
->qid
);
599 if (unlikely(!qid
|| nvme_check_cqid(n
, qid
))) {
600 trace_pci_nvme_err_invalid_del_cq_cqid(qid
);
601 return NVME_INVALID_CQID
| NVME_DNR
;
605 if (unlikely(!QTAILQ_EMPTY(&cq
->sq_list
))) {
606 trace_pci_nvme_err_invalid_del_cq_notempty(qid
);
607 return NVME_INVALID_QUEUE_DEL
;
609 nvme_irq_deassert(n
, cq
);
610 trace_pci_nvme_del_cq(qid
);
615 static void nvme_init_cq(NvmeCQueue
*cq
, NvmeCtrl
*n
, uint64_t dma_addr
,
616 uint16_t cqid
, uint16_t vector
, uint16_t size
, uint16_t irq_enabled
)
621 cq
->dma_addr
= dma_addr
;
623 cq
->irq_enabled
= irq_enabled
;
625 cq
->head
= cq
->tail
= 0;
626 QTAILQ_INIT(&cq
->req_list
);
627 QTAILQ_INIT(&cq
->sq_list
);
628 msix_vector_use(&n
->parent_obj
, cq
->vector
);
630 cq
->timer
= timer_new_ns(QEMU_CLOCK_VIRTUAL
, nvme_post_cqes
, cq
);
633 static uint16_t nvme_create_cq(NvmeCtrl
*n
, NvmeCmd
*cmd
)
636 NvmeCreateCq
*c
= (NvmeCreateCq
*)cmd
;
637 uint16_t cqid
= le16_to_cpu(c
->cqid
);
638 uint16_t vector
= le16_to_cpu(c
->irq_vector
);
639 uint16_t qsize
= le16_to_cpu(c
->qsize
);
640 uint16_t qflags
= le16_to_cpu(c
->cq_flags
);
641 uint64_t prp1
= le64_to_cpu(c
->prp1
);
643 trace_pci_nvme_create_cq(prp1
, cqid
, vector
, qsize
, qflags
,
644 NVME_CQ_FLAGS_IEN(qflags
) != 0);
646 if (unlikely(!cqid
|| !nvme_check_cqid(n
, cqid
))) {
647 trace_pci_nvme_err_invalid_create_cq_cqid(cqid
);
648 return NVME_INVALID_CQID
| NVME_DNR
;
650 if (unlikely(!qsize
|| qsize
> NVME_CAP_MQES(n
->bar
.cap
))) {
651 trace_pci_nvme_err_invalid_create_cq_size(qsize
);
652 return NVME_MAX_QSIZE_EXCEEDED
| NVME_DNR
;
654 if (unlikely(!prp1
)) {
655 trace_pci_nvme_err_invalid_create_cq_addr(prp1
);
656 return NVME_INVALID_FIELD
| NVME_DNR
;
658 if (unlikely(!msix_enabled(&n
->parent_obj
) && vector
)) {
659 trace_pci_nvme_err_invalid_create_cq_vector(vector
);
660 return NVME_INVALID_IRQ_VECTOR
| NVME_DNR
;
662 if (unlikely(vector
> n
->params
.max_ioqpairs
)) {
663 trace_pci_nvme_err_invalid_create_cq_vector(vector
);
664 return NVME_INVALID_IRQ_VECTOR
| NVME_DNR
;
666 if (unlikely(!(NVME_CQ_FLAGS_PC(qflags
)))) {
667 trace_pci_nvme_err_invalid_create_cq_qflags(NVME_CQ_FLAGS_PC(qflags
));
668 return NVME_INVALID_FIELD
| NVME_DNR
;
671 cq
= g_malloc0(sizeof(*cq
));
672 nvme_init_cq(cq
, n
, prp1
, cqid
, vector
, qsize
+ 1,
673 NVME_CQ_FLAGS_IEN(qflags
));
677 static uint16_t nvme_identify_ctrl(NvmeCtrl
*n
, NvmeIdentify
*c
)
679 uint64_t prp1
= le64_to_cpu(c
->prp1
);
680 uint64_t prp2
= le64_to_cpu(c
->prp2
);
682 trace_pci_nvme_identify_ctrl();
684 return nvme_dma_read_prp(n
, (uint8_t *)&n
->id_ctrl
, sizeof(n
->id_ctrl
),
688 static uint16_t nvme_identify_ns(NvmeCtrl
*n
, NvmeIdentify
*c
)
691 uint32_t nsid
= le32_to_cpu(c
->nsid
);
692 uint64_t prp1
= le64_to_cpu(c
->prp1
);
693 uint64_t prp2
= le64_to_cpu(c
->prp2
);
695 trace_pci_nvme_identify_ns(nsid
);
697 if (unlikely(nsid
== 0 || nsid
> n
->num_namespaces
)) {
698 trace_pci_nvme_err_invalid_ns(nsid
, n
->num_namespaces
);
699 return NVME_INVALID_NSID
| NVME_DNR
;
702 ns
= &n
->namespaces
[nsid
- 1];
704 return nvme_dma_read_prp(n
, (uint8_t *)&ns
->id_ns
, sizeof(ns
->id_ns
),
708 static uint16_t nvme_identify_nslist(NvmeCtrl
*n
, NvmeIdentify
*c
)
710 static const int data_len
= NVME_IDENTIFY_DATA_SIZE
;
711 uint32_t min_nsid
= le32_to_cpu(c
->nsid
);
712 uint64_t prp1
= le64_to_cpu(c
->prp1
);
713 uint64_t prp2
= le64_to_cpu(c
->prp2
);
718 trace_pci_nvme_identify_nslist(min_nsid
);
720 list
= g_malloc0(data_len
);
721 for (i
= 0; i
< n
->num_namespaces
; i
++) {
725 list
[j
++] = cpu_to_le32(i
+ 1);
726 if (j
== data_len
/ sizeof(uint32_t)) {
730 ret
= nvme_dma_read_prp(n
, (uint8_t *)list
, data_len
, prp1
, prp2
);
735 static uint16_t nvme_identify(NvmeCtrl
*n
, NvmeCmd
*cmd
)
737 NvmeIdentify
*c
= (NvmeIdentify
*)cmd
;
739 switch (le32_to_cpu(c
->cns
)) {
741 return nvme_identify_ns(n
, c
);
742 case NVME_ID_CNS_CTRL
:
743 return nvme_identify_ctrl(n
, c
);
744 case NVME_ID_CNS_NS_ACTIVE_LIST
:
745 return nvme_identify_nslist(n
, c
);
747 trace_pci_nvme_err_invalid_identify_cns(le32_to_cpu(c
->cns
));
748 return NVME_INVALID_FIELD
| NVME_DNR
;
752 static inline void nvme_set_timestamp(NvmeCtrl
*n
, uint64_t ts
)
754 trace_pci_nvme_setfeat_timestamp(ts
);
756 n
->host_timestamp
= le64_to_cpu(ts
);
757 n
->timestamp_set_qemu_clock_ms
= qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
);
760 static inline uint64_t nvme_get_timestamp(const NvmeCtrl
*n
)
762 uint64_t current_time
= qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
);
763 uint64_t elapsed_time
= current_time
- n
->timestamp_set_qemu_clock_ms
;
765 union nvme_timestamp
{
767 uint64_t timestamp
:48;
775 union nvme_timestamp ts
;
779 * If the sum of the Timestamp value set by the host and the elapsed
780 * time exceeds 2^48, the value returned should be reduced modulo 2^48.
782 ts
.timestamp
= (n
->host_timestamp
+ elapsed_time
) & 0xffffffffffff;
784 /* If the host timestamp is non-zero, set the timestamp origin */
785 ts
.origin
= n
->host_timestamp
? 0x01 : 0x00;
787 trace_pci_nvme_getfeat_timestamp(ts
.all
);
789 return cpu_to_le64(ts
.all
);
792 static uint16_t nvme_get_feature_timestamp(NvmeCtrl
*n
, NvmeCmd
*cmd
)
794 uint64_t prp1
= le64_to_cpu(cmd
->prp1
);
795 uint64_t prp2
= le64_to_cpu(cmd
->prp2
);
797 uint64_t timestamp
= nvme_get_timestamp(n
);
799 return nvme_dma_read_prp(n
, (uint8_t *)×tamp
,
800 sizeof(timestamp
), prp1
, prp2
);
803 static uint16_t nvme_get_feature(NvmeCtrl
*n
, NvmeCmd
*cmd
, NvmeRequest
*req
)
805 uint32_t dw10
= le32_to_cpu(cmd
->cdw10
);
809 case NVME_VOLATILE_WRITE_CACHE
:
810 result
= blk_enable_write_cache(n
->conf
.blk
);
811 trace_pci_nvme_getfeat_vwcache(result
? "enabled" : "disabled");
813 case NVME_NUMBER_OF_QUEUES
:
814 result
= cpu_to_le32((n
->params
.max_ioqpairs
- 1) |
815 ((n
->params
.max_ioqpairs
- 1) << 16));
816 trace_pci_nvme_getfeat_numq(result
);
819 return nvme_get_feature_timestamp(n
, cmd
);
821 trace_pci_nvme_err_invalid_getfeat(dw10
);
822 return NVME_INVALID_FIELD
| NVME_DNR
;
825 req
->cqe
.result
= result
;
829 static uint16_t nvme_set_feature_timestamp(NvmeCtrl
*n
, NvmeCmd
*cmd
)
833 uint64_t prp1
= le64_to_cpu(cmd
->prp1
);
834 uint64_t prp2
= le64_to_cpu(cmd
->prp2
);
836 ret
= nvme_dma_write_prp(n
, (uint8_t *)×tamp
,
837 sizeof(timestamp
), prp1
, prp2
);
838 if (ret
!= NVME_SUCCESS
) {
842 nvme_set_timestamp(n
, timestamp
);
847 static uint16_t nvme_set_feature(NvmeCtrl
*n
, NvmeCmd
*cmd
, NvmeRequest
*req
)
849 uint32_t dw10
= le32_to_cpu(cmd
->cdw10
);
850 uint32_t dw11
= le32_to_cpu(cmd
->cdw11
);
853 case NVME_VOLATILE_WRITE_CACHE
:
854 blk_set_enable_write_cache(n
->conf
.blk
, dw11
& 1);
856 case NVME_NUMBER_OF_QUEUES
:
857 trace_pci_nvme_setfeat_numq((dw11
& 0xFFFF) + 1,
858 ((dw11
>> 16) & 0xFFFF) + 1,
859 n
->params
.max_ioqpairs
,
860 n
->params
.max_ioqpairs
);
861 req
->cqe
.result
= cpu_to_le32((n
->params
.max_ioqpairs
- 1) |
862 ((n
->params
.max_ioqpairs
- 1) << 16));
865 return nvme_set_feature_timestamp(n
, cmd
);
867 trace_pci_nvme_err_invalid_setfeat(dw10
);
868 return NVME_INVALID_FIELD
| NVME_DNR
;
873 static uint16_t nvme_admin_cmd(NvmeCtrl
*n
, NvmeCmd
*cmd
, NvmeRequest
*req
)
875 switch (cmd
->opcode
) {
876 case NVME_ADM_CMD_DELETE_SQ
:
877 return nvme_del_sq(n
, cmd
);
878 case NVME_ADM_CMD_CREATE_SQ
:
879 return nvme_create_sq(n
, cmd
);
880 case NVME_ADM_CMD_DELETE_CQ
:
881 return nvme_del_cq(n
, cmd
);
882 case NVME_ADM_CMD_CREATE_CQ
:
883 return nvme_create_cq(n
, cmd
);
884 case NVME_ADM_CMD_IDENTIFY
:
885 return nvme_identify(n
, cmd
);
886 case NVME_ADM_CMD_SET_FEATURES
:
887 return nvme_set_feature(n
, cmd
, req
);
888 case NVME_ADM_CMD_GET_FEATURES
:
889 return nvme_get_feature(n
, cmd
, req
);
891 trace_pci_nvme_err_invalid_admin_opc(cmd
->opcode
);
892 return NVME_INVALID_OPCODE
| NVME_DNR
;
896 static void nvme_process_sq(void *opaque
)
898 NvmeSQueue
*sq
= opaque
;
899 NvmeCtrl
*n
= sq
->ctrl
;
900 NvmeCQueue
*cq
= n
->cq
[sq
->cqid
];
907 while (!(nvme_sq_empty(sq
) || QTAILQ_EMPTY(&sq
->req_list
))) {
908 addr
= sq
->dma_addr
+ sq
->head
* n
->sqe_size
;
909 nvme_addr_read(n
, addr
, (void *)&cmd
, sizeof(cmd
));
910 nvme_inc_sq_head(sq
);
912 req
= QTAILQ_FIRST(&sq
->req_list
);
913 QTAILQ_REMOVE(&sq
->req_list
, req
, entry
);
914 QTAILQ_INSERT_TAIL(&sq
->out_req_list
, req
, entry
);
915 memset(&req
->cqe
, 0, sizeof(req
->cqe
));
916 req
->cqe
.cid
= cmd
.cid
;
918 status
= sq
->sqid
? nvme_io_cmd(n
, &cmd
, req
) :
919 nvme_admin_cmd(n
, &cmd
, req
);
920 if (status
!= NVME_NO_COMPLETE
) {
921 req
->status
= status
;
922 nvme_enqueue_req_completion(cq
, req
);
927 static void nvme_clear_ctrl(NvmeCtrl
*n
)
931 blk_drain(n
->conf
.blk
);
933 for (i
= 0; i
< n
->params
.max_ioqpairs
+ 1; i
++) {
934 if (n
->sq
[i
] != NULL
) {
935 nvme_free_sq(n
->sq
[i
], n
);
938 for (i
= 0; i
< n
->params
.max_ioqpairs
+ 1; i
++) {
939 if (n
->cq
[i
] != NULL
) {
940 nvme_free_cq(n
->cq
[i
], n
);
944 blk_flush(n
->conf
.blk
);
948 static int nvme_start_ctrl(NvmeCtrl
*n
)
950 uint32_t page_bits
= NVME_CC_MPS(n
->bar
.cc
) + 12;
951 uint32_t page_size
= 1 << page_bits
;
953 if (unlikely(n
->cq
[0])) {
954 trace_pci_nvme_err_startfail_cq();
957 if (unlikely(n
->sq
[0])) {
958 trace_pci_nvme_err_startfail_sq();
961 if (unlikely(!n
->bar
.asq
)) {
962 trace_pci_nvme_err_startfail_nbarasq();
965 if (unlikely(!n
->bar
.acq
)) {
966 trace_pci_nvme_err_startfail_nbaracq();
969 if (unlikely(n
->bar
.asq
& (page_size
- 1))) {
970 trace_pci_nvme_err_startfail_asq_misaligned(n
->bar
.asq
);
973 if (unlikely(n
->bar
.acq
& (page_size
- 1))) {
974 trace_pci_nvme_err_startfail_acq_misaligned(n
->bar
.acq
);
977 if (unlikely(NVME_CC_MPS(n
->bar
.cc
) <
978 NVME_CAP_MPSMIN(n
->bar
.cap
))) {
979 trace_pci_nvme_err_startfail_page_too_small(
980 NVME_CC_MPS(n
->bar
.cc
),
981 NVME_CAP_MPSMIN(n
->bar
.cap
));
984 if (unlikely(NVME_CC_MPS(n
->bar
.cc
) >
985 NVME_CAP_MPSMAX(n
->bar
.cap
))) {
986 trace_pci_nvme_err_startfail_page_too_large(
987 NVME_CC_MPS(n
->bar
.cc
),
988 NVME_CAP_MPSMAX(n
->bar
.cap
));
991 if (unlikely(NVME_CC_IOCQES(n
->bar
.cc
) <
992 NVME_CTRL_CQES_MIN(n
->id_ctrl
.cqes
))) {
993 trace_pci_nvme_err_startfail_cqent_too_small(
994 NVME_CC_IOCQES(n
->bar
.cc
),
995 NVME_CTRL_CQES_MIN(n
->bar
.cap
));
998 if (unlikely(NVME_CC_IOCQES(n
->bar
.cc
) >
999 NVME_CTRL_CQES_MAX(n
->id_ctrl
.cqes
))) {
1000 trace_pci_nvme_err_startfail_cqent_too_large(
1001 NVME_CC_IOCQES(n
->bar
.cc
),
1002 NVME_CTRL_CQES_MAX(n
->bar
.cap
));
1005 if (unlikely(NVME_CC_IOSQES(n
->bar
.cc
) <
1006 NVME_CTRL_SQES_MIN(n
->id_ctrl
.sqes
))) {
1007 trace_pci_nvme_err_startfail_sqent_too_small(
1008 NVME_CC_IOSQES(n
->bar
.cc
),
1009 NVME_CTRL_SQES_MIN(n
->bar
.cap
));
1012 if (unlikely(NVME_CC_IOSQES(n
->bar
.cc
) >
1013 NVME_CTRL_SQES_MAX(n
->id_ctrl
.sqes
))) {
1014 trace_pci_nvme_err_startfail_sqent_too_large(
1015 NVME_CC_IOSQES(n
->bar
.cc
),
1016 NVME_CTRL_SQES_MAX(n
->bar
.cap
));
1019 if (unlikely(!NVME_AQA_ASQS(n
->bar
.aqa
))) {
1020 trace_pci_nvme_err_startfail_asqent_sz_zero();
1023 if (unlikely(!NVME_AQA_ACQS(n
->bar
.aqa
))) {
1024 trace_pci_nvme_err_startfail_acqent_sz_zero();
1028 n
->page_bits
= page_bits
;
1029 n
->page_size
= page_size
;
1030 n
->max_prp_ents
= n
->page_size
/ sizeof(uint64_t);
1031 n
->cqe_size
= 1 << NVME_CC_IOCQES(n
->bar
.cc
);
1032 n
->sqe_size
= 1 << NVME_CC_IOSQES(n
->bar
.cc
);
1033 nvme_init_cq(&n
->admin_cq
, n
, n
->bar
.acq
, 0, 0,
1034 NVME_AQA_ACQS(n
->bar
.aqa
) + 1, 1);
1035 nvme_init_sq(&n
->admin_sq
, n
, n
->bar
.asq
, 0, 0,
1036 NVME_AQA_ASQS(n
->bar
.aqa
) + 1);
1038 nvme_set_timestamp(n
, 0ULL);
1043 static void nvme_write_bar(NvmeCtrl
*n
, hwaddr offset
, uint64_t data
,
1046 if (unlikely(offset
& (sizeof(uint32_t) - 1))) {
1047 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_misaligned32
,
1048 "MMIO write not 32-bit aligned,"
1049 " offset=0x%"PRIx64
"", offset
);
1050 /* should be ignored, fall through for now */
1053 if (unlikely(size
< sizeof(uint32_t))) {
1054 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_toosmall
,
1055 "MMIO write smaller than 32-bits,"
1056 " offset=0x%"PRIx64
", size=%u",
1058 /* should be ignored, fall through for now */
1062 case 0xc: /* INTMS */
1063 if (unlikely(msix_enabled(&(n
->parent_obj
)))) {
1064 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix
,
1065 "undefined access to interrupt mask set"
1066 " when MSI-X is enabled");
1067 /* should be ignored, fall through for now */
1069 n
->bar
.intms
|= data
& 0xffffffff;
1070 n
->bar
.intmc
= n
->bar
.intms
;
1071 trace_pci_nvme_mmio_intm_set(data
& 0xffffffff, n
->bar
.intmc
);
1074 case 0x10: /* INTMC */
1075 if (unlikely(msix_enabled(&(n
->parent_obj
)))) {
1076 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix
,
1077 "undefined access to interrupt mask clr"
1078 " when MSI-X is enabled");
1079 /* should be ignored, fall through for now */
1081 n
->bar
.intms
&= ~(data
& 0xffffffff);
1082 n
->bar
.intmc
= n
->bar
.intms
;
1083 trace_pci_nvme_mmio_intm_clr(data
& 0xffffffff, n
->bar
.intmc
);
1087 trace_pci_nvme_mmio_cfg(data
& 0xffffffff);
1088 /* Windows first sends data, then sends enable bit */
1089 if (!NVME_CC_EN(data
) && !NVME_CC_EN(n
->bar
.cc
) &&
1090 !NVME_CC_SHN(data
) && !NVME_CC_SHN(n
->bar
.cc
))
1095 if (NVME_CC_EN(data
) && !NVME_CC_EN(n
->bar
.cc
)) {
1097 if (unlikely(nvme_start_ctrl(n
))) {
1098 trace_pci_nvme_err_startfail();
1099 n
->bar
.csts
= NVME_CSTS_FAILED
;
1101 trace_pci_nvme_mmio_start_success();
1102 n
->bar
.csts
= NVME_CSTS_READY
;
1104 } else if (!NVME_CC_EN(data
) && NVME_CC_EN(n
->bar
.cc
)) {
1105 trace_pci_nvme_mmio_stopped();
1107 n
->bar
.csts
&= ~NVME_CSTS_READY
;
1109 if (NVME_CC_SHN(data
) && !(NVME_CC_SHN(n
->bar
.cc
))) {
1110 trace_pci_nvme_mmio_shutdown_set();
1113 n
->bar
.csts
|= NVME_CSTS_SHST_COMPLETE
;
1114 } else if (!NVME_CC_SHN(data
) && NVME_CC_SHN(n
->bar
.cc
)) {
1115 trace_pci_nvme_mmio_shutdown_cleared();
1116 n
->bar
.csts
&= ~NVME_CSTS_SHST_COMPLETE
;
1120 case 0x1C: /* CSTS */
1121 if (data
& (1 << 4)) {
1122 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ssreset_w1c_unsupported
,
1123 "attempted to W1C CSTS.NSSRO"
1124 " but CAP.NSSRS is zero (not supported)");
1125 } else if (data
!= 0) {
1126 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ro_csts
,
1127 "attempted to set a read only bit"
1128 " of controller status");
1131 case 0x20: /* NSSR */
1132 if (data
== 0x4E564D65) {
1133 trace_pci_nvme_ub_mmiowr_ssreset_unsupported();
1135 /* The spec says that writes of other values have no effect */
1139 case 0x24: /* AQA */
1140 n
->bar
.aqa
= data
& 0xffffffff;
1141 trace_pci_nvme_mmio_aqattr(data
& 0xffffffff);
1143 case 0x28: /* ASQ */
1145 trace_pci_nvme_mmio_asqaddr(data
);
1147 case 0x2c: /* ASQ hi */
1148 n
->bar
.asq
|= data
<< 32;
1149 trace_pci_nvme_mmio_asqaddr_hi(data
, n
->bar
.asq
);
1151 case 0x30: /* ACQ */
1152 trace_pci_nvme_mmio_acqaddr(data
);
1155 case 0x34: /* ACQ hi */
1156 n
->bar
.acq
|= data
<< 32;
1157 trace_pci_nvme_mmio_acqaddr_hi(data
, n
->bar
.acq
);
1159 case 0x38: /* CMBLOC */
1160 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbloc_reserved
,
1161 "invalid write to reserved CMBLOC"
1162 " when CMBSZ is zero, ignored");
1164 case 0x3C: /* CMBSZ */
1165 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbsz_readonly
,
1166 "invalid write to read only CMBSZ, ignored");
1168 case 0xE00: /* PMRCAP */
1169 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrcap_readonly
,
1170 "invalid write to PMRCAP register, ignored");
1172 case 0xE04: /* TODO PMRCTL */
1174 case 0xE08: /* PMRSTS */
1175 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrsts_readonly
,
1176 "invalid write to PMRSTS register, ignored");
1178 case 0xE0C: /* PMREBS */
1179 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrebs_readonly
,
1180 "invalid write to PMREBS register, ignored");
1182 case 0xE10: /* PMRSWTP */
1183 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrswtp_readonly
,
1184 "invalid write to PMRSWTP register, ignored");
1186 case 0xE14: /* TODO PMRMSC */
1189 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_invalid
,
1190 "invalid MMIO write,"
1191 " offset=0x%"PRIx64
", data=%"PRIx64
"",
1197 static uint64_t nvme_mmio_read(void *opaque
, hwaddr addr
, unsigned size
)
1199 NvmeCtrl
*n
= (NvmeCtrl
*)opaque
;
1200 uint8_t *ptr
= (uint8_t *)&n
->bar
;
1203 if (unlikely(addr
& (sizeof(uint32_t) - 1))) {
1204 NVME_GUEST_ERR(pci_nvme_ub_mmiord_misaligned32
,
1205 "MMIO read not 32-bit aligned,"
1206 " offset=0x%"PRIx64
"", addr
);
1207 /* should RAZ, fall through for now */
1208 } else if (unlikely(size
< sizeof(uint32_t))) {
1209 NVME_GUEST_ERR(pci_nvme_ub_mmiord_toosmall
,
1210 "MMIO read smaller than 32-bits,"
1211 " offset=0x%"PRIx64
"", addr
);
1212 /* should RAZ, fall through for now */
1215 if (addr
< sizeof(n
->bar
)) {
1217 * When PMRWBM bit 1 is set then read from
1218 * from PMRSTS should ensure prior writes
1219 * made it to persistent media
1221 if (addr
== 0xE08 &&
1222 (NVME_PMRCAP_PMRWBM(n
->bar
.pmrcap
) & 0x02)) {
1223 memory_region_msync(&n
->pmrdev
->mr
, 0, n
->pmrdev
->size
);
1225 memcpy(&val
, ptr
+ addr
, size
);
1227 NVME_GUEST_ERR(pci_nvme_ub_mmiord_invalid_ofs
,
1228 "MMIO read beyond last register,"
1229 " offset=0x%"PRIx64
", returning 0", addr
);
1235 static void nvme_process_db(NvmeCtrl
*n
, hwaddr addr
, int val
)
1239 if (unlikely(addr
& ((1 << 2) - 1))) {
1240 NVME_GUEST_ERR(pci_nvme_ub_db_wr_misaligned
,
1241 "doorbell write not 32-bit aligned,"
1242 " offset=0x%"PRIx64
", ignoring", addr
);
1246 if (((addr
- 0x1000) >> 2) & 1) {
1247 /* Completion queue doorbell write */
1249 uint16_t new_head
= val
& 0xffff;
1253 qid
= (addr
- (0x1000 + (1 << 2))) >> 3;
1254 if (unlikely(nvme_check_cqid(n
, qid
))) {
1255 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cq
,
1256 "completion queue doorbell write"
1257 " for nonexistent queue,"
1258 " sqid=%"PRIu32
", ignoring", qid
);
1263 if (unlikely(new_head
>= cq
->size
)) {
1264 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cqhead
,
1265 "completion queue doorbell write value"
1266 " beyond queue size, sqid=%"PRIu32
","
1267 " new_head=%"PRIu16
", ignoring",
1272 start_sqs
= nvme_cq_full(cq
) ? 1 : 0;
1273 cq
->head
= new_head
;
1276 QTAILQ_FOREACH(sq
, &cq
->sq_list
, entry
) {
1277 timer_mod(sq
->timer
, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL
) + 500);
1279 timer_mod(cq
->timer
, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL
) + 500);
1282 if (cq
->tail
== cq
->head
) {
1283 nvme_irq_deassert(n
, cq
);
1286 /* Submission queue doorbell write */
1288 uint16_t new_tail
= val
& 0xffff;
1291 qid
= (addr
- 0x1000) >> 3;
1292 if (unlikely(nvme_check_sqid(n
, qid
))) {
1293 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sq
,
1294 "submission queue doorbell write"
1295 " for nonexistent queue,"
1296 " sqid=%"PRIu32
", ignoring", qid
);
1301 if (unlikely(new_tail
>= sq
->size
)) {
1302 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sqtail
,
1303 "submission queue doorbell write value"
1304 " beyond queue size, sqid=%"PRIu32
","
1305 " new_tail=%"PRIu16
", ignoring",
1310 sq
->tail
= new_tail
;
1311 timer_mod(sq
->timer
, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL
) + 500);
1315 static void nvme_mmio_write(void *opaque
, hwaddr addr
, uint64_t data
,
1318 NvmeCtrl
*n
= (NvmeCtrl
*)opaque
;
1319 if (addr
< sizeof(n
->bar
)) {
1320 nvme_write_bar(n
, addr
, data
, size
);
1321 } else if (addr
>= 0x1000) {
1322 nvme_process_db(n
, addr
, data
);
1326 static const MemoryRegionOps nvme_mmio_ops
= {
1327 .read
= nvme_mmio_read
,
1328 .write
= nvme_mmio_write
,
1329 .endianness
= DEVICE_LITTLE_ENDIAN
,
1331 .min_access_size
= 2,
1332 .max_access_size
= 8,
1336 static void nvme_cmb_write(void *opaque
, hwaddr addr
, uint64_t data
,
1339 NvmeCtrl
*n
= (NvmeCtrl
*)opaque
;
1340 stn_le_p(&n
->cmbuf
[addr
], size
, data
);
1343 static uint64_t nvme_cmb_read(void *opaque
, hwaddr addr
, unsigned size
)
1345 NvmeCtrl
*n
= (NvmeCtrl
*)opaque
;
1346 return ldn_le_p(&n
->cmbuf
[addr
], size
);
1349 static const MemoryRegionOps nvme_cmb_ops
= {
1350 .read
= nvme_cmb_read
,
1351 .write
= nvme_cmb_write
,
1352 .endianness
= DEVICE_LITTLE_ENDIAN
,
1354 .min_access_size
= 1,
1355 .max_access_size
= 8,
1359 static void nvme_check_constraints(NvmeCtrl
*n
, Error
**errp
)
1361 NvmeParams
*params
= &n
->params
;
1363 if (params
->num_queues
) {
1364 warn_report("num_queues is deprecated; please use max_ioqpairs "
1367 params
->max_ioqpairs
= params
->num_queues
- 1;
1370 if (params
->max_ioqpairs
< 1 ||
1371 params
->max_ioqpairs
> PCI_MSIX_FLAGS_QSIZE
) {
1372 error_setg(errp
, "max_ioqpairs must be between 1 and %d",
1373 PCI_MSIX_FLAGS_QSIZE
);
1378 error_setg(errp
, "drive property not set");
1382 if (!params
->serial
) {
1383 error_setg(errp
, "serial property not set");
1387 if (!n
->params
.cmb_size_mb
&& n
->pmrdev
) {
1388 if (host_memory_backend_is_mapped(n
->pmrdev
)) {
1389 char *path
= object_get_canonical_path_component(OBJECT(n
->pmrdev
));
1390 error_setg(errp
, "can't use already busy memdev: %s", path
);
1395 if (!is_power_of_2(n
->pmrdev
->size
)) {
1396 error_setg(errp
, "pmr backend size needs to be power of 2 in size");
1400 host_memory_backend_set_mapped(n
->pmrdev
, true);
1404 static void nvme_init_state(NvmeCtrl
*n
)
1406 n
->num_namespaces
= 1;
1407 /* add one to max_ioqpairs to account for the admin queue pair */
1408 n
->reg_size
= pow2ceil(NVME_REG_SIZE
+
1409 2 * (n
->params
.max_ioqpairs
+ 1) * NVME_DB_SIZE
);
1410 n
->namespaces
= g_new0(NvmeNamespace
, n
->num_namespaces
);
1411 n
->sq
= g_new0(NvmeSQueue
*, n
->params
.max_ioqpairs
+ 1);
1412 n
->cq
= g_new0(NvmeCQueue
*, n
->params
.max_ioqpairs
+ 1);
1415 static void nvme_init_blk(NvmeCtrl
*n
, Error
**errp
)
1417 blkconf_blocksizes(&n
->conf
);
1418 blkconf_apply_backend_options(&n
->conf
, blk_is_read_only(n
->conf
.blk
),
1422 static void nvme_init_namespace(NvmeCtrl
*n
, NvmeNamespace
*ns
, Error
**errp
)
1425 NvmeIdNs
*id_ns
= &ns
->id_ns
;
1427 bs_size
= blk_getlength(n
->conf
.blk
);
1429 error_setg_errno(errp
, -bs_size
, "could not get backing file size");
1433 n
->ns_size
= bs_size
;
1435 id_ns
->lbaf
[0].ds
= BDRV_SECTOR_BITS
;
1436 id_ns
->nsze
= cpu_to_le64(nvme_ns_nlbas(n
, ns
));
1438 /* no thin provisioning */
1439 id_ns
->ncap
= id_ns
->nsze
;
1440 id_ns
->nuse
= id_ns
->ncap
;
1443 static void nvme_init_cmb(NvmeCtrl
*n
, PCIDevice
*pci_dev
)
1445 NVME_CMBLOC_SET_BIR(n
->bar
.cmbloc
, NVME_CMB_BIR
);
1446 NVME_CMBLOC_SET_OFST(n
->bar
.cmbloc
, 0);
1448 NVME_CMBSZ_SET_SQS(n
->bar
.cmbsz
, 1);
1449 NVME_CMBSZ_SET_CQS(n
->bar
.cmbsz
, 0);
1450 NVME_CMBSZ_SET_LISTS(n
->bar
.cmbsz
, 0);
1451 NVME_CMBSZ_SET_RDS(n
->bar
.cmbsz
, 1);
1452 NVME_CMBSZ_SET_WDS(n
->bar
.cmbsz
, 1);
1453 NVME_CMBSZ_SET_SZU(n
->bar
.cmbsz
, 2); /* MBs */
1454 NVME_CMBSZ_SET_SZ(n
->bar
.cmbsz
, n
->params
.cmb_size_mb
);
1456 n
->cmbuf
= g_malloc0(NVME_CMBSZ_GETSIZE(n
->bar
.cmbsz
));
1457 memory_region_init_io(&n
->ctrl_mem
, OBJECT(n
), &nvme_cmb_ops
, n
,
1458 "nvme-cmb", NVME_CMBSZ_GETSIZE(n
->bar
.cmbsz
));
1459 pci_register_bar(pci_dev
, NVME_CMBLOC_BIR(n
->bar
.cmbloc
),
1460 PCI_BASE_ADDRESS_SPACE_MEMORY
|
1461 PCI_BASE_ADDRESS_MEM_TYPE_64
|
1462 PCI_BASE_ADDRESS_MEM_PREFETCH
, &n
->ctrl_mem
);
1465 static void nvme_init_pmr(NvmeCtrl
*n
, PCIDevice
*pci_dev
)
1467 /* Controller Capabilities register */
1468 NVME_CAP_SET_PMRS(n
->bar
.cap
, 1);
1470 /* PMR Capabities register */
1472 NVME_PMRCAP_SET_RDS(n
->bar
.pmrcap
, 0);
1473 NVME_PMRCAP_SET_WDS(n
->bar
.pmrcap
, 0);
1474 NVME_PMRCAP_SET_BIR(n
->bar
.pmrcap
, NVME_PMR_BIR
);
1475 NVME_PMRCAP_SET_PMRTU(n
->bar
.pmrcap
, 0);
1476 /* Turn on bit 1 support */
1477 NVME_PMRCAP_SET_PMRWBM(n
->bar
.pmrcap
, 0x02);
1478 NVME_PMRCAP_SET_PMRTO(n
->bar
.pmrcap
, 0);
1479 NVME_PMRCAP_SET_CMSS(n
->bar
.pmrcap
, 0);
1481 /* PMR Control register */
1483 NVME_PMRCTL_SET_EN(n
->bar
.pmrctl
, 0);
1485 /* PMR Status register */
1487 NVME_PMRSTS_SET_ERR(n
->bar
.pmrsts
, 0);
1488 NVME_PMRSTS_SET_NRDY(n
->bar
.pmrsts
, 0);
1489 NVME_PMRSTS_SET_HSTS(n
->bar
.pmrsts
, 0);
1490 NVME_PMRSTS_SET_CBAI(n
->bar
.pmrsts
, 0);
1492 /* PMR Elasticity Buffer Size register */
1494 NVME_PMREBS_SET_PMRSZU(n
->bar
.pmrebs
, 0);
1495 NVME_PMREBS_SET_RBB(n
->bar
.pmrebs
, 0);
1496 NVME_PMREBS_SET_PMRWBZ(n
->bar
.pmrebs
, 0);
1498 /* PMR Sustained Write Throughput register */
1500 NVME_PMRSWTP_SET_PMRSWTU(n
->bar
.pmrswtp
, 0);
1501 NVME_PMRSWTP_SET_PMRSWTV(n
->bar
.pmrswtp
, 0);
1503 /* PMR Memory Space Control register */
1505 NVME_PMRMSC_SET_CMSE(n
->bar
.pmrmsc
, 0);
1506 NVME_PMRMSC_SET_CBA(n
->bar
.pmrmsc
, 0);
1508 pci_register_bar(pci_dev
, NVME_PMRCAP_BIR(n
->bar
.pmrcap
),
1509 PCI_BASE_ADDRESS_SPACE_MEMORY
|
1510 PCI_BASE_ADDRESS_MEM_TYPE_64
|
1511 PCI_BASE_ADDRESS_MEM_PREFETCH
, &n
->pmrdev
->mr
);
1514 static void nvme_init_pci(NvmeCtrl
*n
, PCIDevice
*pci_dev
)
1516 uint8_t *pci_conf
= pci_dev
->config
;
1518 pci_conf
[PCI_INTERRUPT_PIN
] = 1;
1519 pci_config_set_prog_interface(pci_conf
, 0x2);
1520 pci_config_set_class(pci_conf
, PCI_CLASS_STORAGE_EXPRESS
);
1521 pcie_endpoint_cap_init(pci_dev
, 0x80);
1523 memory_region_init_io(&n
->iomem
, OBJECT(n
), &nvme_mmio_ops
, n
, "nvme",
1525 pci_register_bar(pci_dev
, 0, PCI_BASE_ADDRESS_SPACE_MEMORY
|
1526 PCI_BASE_ADDRESS_MEM_TYPE_64
, &n
->iomem
);
1527 msix_init_exclusive_bar(pci_dev
, n
->params
.max_ioqpairs
+ 1, 4, NULL
);
1529 if (n
->params
.cmb_size_mb
) {
1530 nvme_init_cmb(n
, pci_dev
);
1531 } else if (n
->pmrdev
) {
1532 nvme_init_pmr(n
, pci_dev
);
1536 static void nvme_init_ctrl(NvmeCtrl
*n
, PCIDevice
*pci_dev
)
1538 NvmeIdCtrl
*id
= &n
->id_ctrl
;
1539 uint8_t *pci_conf
= pci_dev
->config
;
1541 id
->vid
= cpu_to_le16(pci_get_word(pci_conf
+ PCI_VENDOR_ID
));
1542 id
->ssvid
= cpu_to_le16(pci_get_word(pci_conf
+ PCI_SUBSYSTEM_VENDOR_ID
));
1543 strpadcpy((char *)id
->mn
, sizeof(id
->mn
), "QEMU NVMe Ctrl", ' ');
1544 strpadcpy((char *)id
->fr
, sizeof(id
->fr
), "1.0", ' ');
1545 strpadcpy((char *)id
->sn
, sizeof(id
->sn
), n
->params
.serial
, ' ');
1550 id
->oacs
= cpu_to_le16(0);
1553 id
->sqes
= (0x6 << 4) | 0x6;
1554 id
->cqes
= (0x4 << 4) | 0x4;
1555 id
->nn
= cpu_to_le32(n
->num_namespaces
);
1556 id
->oncs
= cpu_to_le16(NVME_ONCS_WRITE_ZEROS
| NVME_ONCS_TIMESTAMP
);
1557 id
->psd
[0].mp
= cpu_to_le16(0x9c4);
1558 id
->psd
[0].enlat
= cpu_to_le32(0x10);
1559 id
->psd
[0].exlat
= cpu_to_le32(0x4);
1560 if (blk_enable_write_cache(n
->conf
.blk
)) {
1565 NVME_CAP_SET_MQES(n
->bar
.cap
, 0x7ff);
1566 NVME_CAP_SET_CQR(n
->bar
.cap
, 1);
1567 NVME_CAP_SET_TO(n
->bar
.cap
, 0xf);
1568 NVME_CAP_SET_CSS(n
->bar
.cap
, 1);
1569 NVME_CAP_SET_MPSMAX(n
->bar
.cap
, 4);
1571 n
->bar
.vs
= 0x00010200;
1572 n
->bar
.intmc
= n
->bar
.intms
= 0;
1575 static void nvme_realize(PCIDevice
*pci_dev
, Error
**errp
)
1577 NvmeCtrl
*n
= NVME(pci_dev
);
1578 Error
*local_err
= NULL
;
1582 nvme_check_constraints(n
, &local_err
);
1584 error_propagate(errp
, local_err
);
1589 nvme_init_blk(n
, &local_err
);
1591 error_propagate(errp
, local_err
);
1595 nvme_init_pci(n
, pci_dev
);
1596 nvme_init_ctrl(n
, pci_dev
);
1598 for (i
= 0; i
< n
->num_namespaces
; i
++) {
1599 nvme_init_namespace(n
, &n
->namespaces
[i
], &local_err
);
1601 error_propagate(errp
, local_err
);
1607 static void nvme_exit(PCIDevice
*pci_dev
)
1609 NvmeCtrl
*n
= NVME(pci_dev
);
1612 g_free(n
->namespaces
);
1616 if (n
->params
.cmb_size_mb
) {
1621 host_memory_backend_set_mapped(n
->pmrdev
, false);
1623 msix_uninit_exclusive_bar(pci_dev
);
1626 static Property nvme_props
[] = {
1627 DEFINE_BLOCK_PROPERTIES(NvmeCtrl
, conf
),
1628 DEFINE_PROP_LINK("pmrdev", NvmeCtrl
, pmrdev
, TYPE_MEMORY_BACKEND
,
1629 HostMemoryBackend
*),
1630 DEFINE_PROP_STRING("serial", NvmeCtrl
, params
.serial
),
1631 DEFINE_PROP_UINT32("cmb_size_mb", NvmeCtrl
, params
.cmb_size_mb
, 0),
1632 DEFINE_PROP_UINT32("num_queues", NvmeCtrl
, params
.num_queues
, 0),
1633 DEFINE_PROP_UINT32("max_ioqpairs", NvmeCtrl
, params
.max_ioqpairs
, 64),
1634 DEFINE_PROP_END_OF_LIST(),
1637 static const VMStateDescription nvme_vmstate
= {
1642 static void nvme_class_init(ObjectClass
*oc
, void *data
)
1644 DeviceClass
*dc
= DEVICE_CLASS(oc
);
1645 PCIDeviceClass
*pc
= PCI_DEVICE_CLASS(oc
);
1647 pc
->realize
= nvme_realize
;
1648 pc
->exit
= nvme_exit
;
1649 pc
->class_id
= PCI_CLASS_STORAGE_EXPRESS
;
1650 pc
->vendor_id
= PCI_VENDOR_ID_INTEL
;
1651 pc
->device_id
= 0x5845;
1654 set_bit(DEVICE_CATEGORY_STORAGE
, dc
->categories
);
1655 dc
->desc
= "Non-Volatile Memory Express";
1656 device_class_set_props(dc
, nvme_props
);
1657 dc
->vmsd
= &nvme_vmstate
;
1660 static void nvme_instance_init(Object
*obj
)
1662 NvmeCtrl
*s
= NVME(obj
);
1664 device_add_bootindex_property(obj
, &s
->conf
.bootindex
,
1665 "bootindex", "/namespace@1,0",
1669 static const TypeInfo nvme_info
= {
1671 .parent
= TYPE_PCI_DEVICE
,
1672 .instance_size
= sizeof(NvmeCtrl
),
1673 .class_init
= nvme_class_init
,
1674 .instance_init
= nvme_instance_init
,
1675 .interfaces
= (InterfaceInfo
[]) {
1676 { INTERFACE_PCIE_DEVICE
},
1681 static void nvme_register_types(void)
1683 type_register_static(&nvme_info
);
1686 type_init(nvme_register_types
)