hw/block/nvme.c

   1 /*
   2  * QEMU NVM Express Controller
   3  *
   4  * Copyright (c) 2012, Intel Corporation
   5  *
   6  * Written by Keith Busch <keith.busch@intel.com>
   7  *
   8  * This code is licensed under the GNU GPL v2 or later.
   9  */
  10
  11 /**
  12  * Reference Specs: http://www.nvmexpress.org, 1.2, 1.1, 1.0e
  13  *
  14  *  http://www.nvmexpress.org/resources/
  15  */
  16
  17 /**
  18  * Usage: add options:
  19  *      -drive file=<file>,if=none,id=<drive_id>
  20  *      -device nvme,drive=<drive_id>,serial=<serial>,id=<id[optional]>, \
  21  *              cmb_size_mb=<cmb_size_mb[optional]>, \
  22  *              [pmrdev=<mem_backend_file_id>,] \
  23  *              num_queues=<N[optional]>
  24  *
  25  * Note cmb_size_mb denotes size of CMB in MB. CMB is assumed to be at
  26  * offset 0 in BAR2 and supports only WDS, RDS and SQS for now.
  27  *
  28  * cmb_size_mb= and pmrdev= options are mutually exclusive due to limitation
  29  * in available BAR's. cmb_size_mb= will take precedence over pmrdev= when
  30  * both provided.
  31  * Enabling pmr emulation can be achieved by pointing to memory-backend-file.
  32  * For example:
  33  * -object memory-backend-file,id=<mem_id>,share=on,mem-path=<file_path>, \
  34  *  size=<size> .... -device nvme,...,pmrdev=<mem_id>
  35  */
  36
  37 #include "qemu/osdep.h"
  38 #include "qemu/units.h"
  39 #include "hw/block/block.h"
  40 #include "hw/pci/msix.h"
  41 #include "hw/pci/pci.h"
  42 #include "hw/qdev-properties.h"
  43 #include "migration/vmstate.h"
  44 #include "sysemu/sysemu.h"
  45 #include "qapi/error.h"
  46 #include "qapi/visitor.h"
  47 #include "sysemu/hostmem.h"
  48 #include "sysemu/block-backend.h"
  49 #include "exec/ram_addr.h"
  50
  51 #include "qemu/log.h"
  52 #include "qemu/module.h"
  53 #include "qemu/cutils.h"
  54 #include "trace.h"
  55 #include "nvme.h"
  56
  57 #define NVME_GUEST_ERR(trace, fmt, ...) \
  58     do { \
  59         (trace_##trace)(__VA_ARGS__); \
  60         qemu_log_mask(LOG_GUEST_ERROR, #trace \
  61             " in %s: " fmt "\n", __func__, ## __VA_ARGS__); \
  62     } while (0)
  63
  64 static void nvme_process_sq(void *opaque);
  65
  66 static void nvme_addr_read(NvmeCtrl *n, hwaddr addr, void *buf, int size)
  67 {
  68     if (n->cmbsz && addr >= n->ctrl_mem.addr &&
  69                 addr < (n->ctrl_mem.addr + int128_get64(n->ctrl_mem.size))) {
  70         memcpy(buf, (void *)&n->cmbuf[addr - n->ctrl_mem.addr], size);
  71     } else {
  72         pci_dma_read(&n->parent_obj, addr, buf, size);
  73     }
  74 }
  75
  76 static int nvme_check_sqid(NvmeCtrl *n, uint16_t sqid)
  77 {
  78     return sqid < n->num_queues && n->sq[sqid] != NULL ? 0 : -1;
  79 }
  80
  81 static int nvme_check_cqid(NvmeCtrl *n, uint16_t cqid)
  82 {
  83     return cqid < n->num_queues && n->cq[cqid] != NULL ? 0 : -1;
  84 }
  85
  86 static void nvme_inc_cq_tail(NvmeCQueue *cq)
  87 {
  88     cq->tail++;
  89     if (cq->tail >= cq->size) {
  90         cq->tail = 0;
  91         cq->phase = !cq->phase;
  92     }
  93 }
  94
  95 static void nvme_inc_sq_head(NvmeSQueue *sq)
  96 {
  97     sq->head = (sq->head + 1) % sq->size;
  98 }
  99
 100 static uint8_t nvme_cq_full(NvmeCQueue *cq)
 101 {
 102     return (cq->tail + 1) % cq->size == cq->head;
 103 }
 104
 105 static uint8_t nvme_sq_empty(NvmeSQueue *sq)
 106 {
 107     return sq->head == sq->tail;
 108 }
 109
 110 static void nvme_irq_check(NvmeCtrl *n)
 111 {
 112     if (msix_enabled(&(n->parent_obj))) {
 113         return;
 114     }
 115     if (~n->bar.intms & n->irq_status) {
 116         pci_irq_assert(&n->parent_obj);
 117     } else {
 118         pci_irq_deassert(&n->parent_obj);
 119     }
 120 }
 121
 122 static void nvme_irq_assert(NvmeCtrl *n, NvmeCQueue *cq)
 123 {
 124     if (cq->irq_enabled) {
 125         if (msix_enabled(&(n->parent_obj))) {
 126             trace_nvme_irq_msix(cq->vector);
 127             msix_notify(&(n->parent_obj), cq->vector);
 128         } else {
 129             trace_nvme_irq_pin();
 130             assert(cq->cqid < 64);
 131             n->irq_status |= 1 << cq->cqid;
 132             nvme_irq_check(n);
 133         }
 134     } else {
 135         trace_nvme_irq_masked();
 136     }
 137 }
 138
 139 static void nvme_irq_deassert(NvmeCtrl *n, NvmeCQueue *cq)
 140 {
 141     if (cq->irq_enabled) {
 142         if (msix_enabled(&(n->parent_obj))) {
 143             return;
 144         } else {
 145             assert(cq->cqid < 64);
 146             n->irq_status &= ~(1 << cq->cqid);
 147             nvme_irq_check(n);
 148         }
 149     }
 150 }
 151
 152 static uint16_t nvme_map_prp(QEMUSGList *qsg, QEMUIOVector *iov, uint64_t prp1,
 153                              uint64_t prp2, uint32_t len, NvmeCtrl *n)
 154 {
 155     hwaddr trans_len = n->page_size - (prp1 % n->page_size);
 156     trans_len = MIN(len, trans_len);
 157     int num_prps = (len >> n->page_bits) + 1;
 158
 159     if (unlikely(!prp1)) {
 160         trace_nvme_err_invalid_prp();
 161         return NVME_INVALID_FIELD | NVME_DNR;
 162     } else if (n->cmbsz && prp1 >= n->ctrl_mem.addr &&
 163                prp1 < n->ctrl_mem.addr + int128_get64(n->ctrl_mem.size)) {
 164         qsg->nsg = 0;
 165         qemu_iovec_init(iov, num_prps);
 166         qemu_iovec_add(iov, (void *)&n->cmbuf[prp1 - n->ctrl_mem.addr], trans_len);
 167     } else {
 168         pci_dma_sglist_init(qsg, &n->parent_obj, num_prps);
 169         qemu_sglist_add(qsg, prp1, trans_len);
 170     }
 171     len -= trans_len;
 172     if (len) {
 173         if (unlikely(!prp2)) {
 174             trace_nvme_err_invalid_prp2_missing();
 175             goto unmap;
 176         }
 177         if (len > n->page_size) {
 178             uint64_t prp_list[n->max_prp_ents];
 179             uint32_t nents, prp_trans;
 180             int i = 0;
 181
 182             nents = (len + n->page_size - 1) >> n->page_bits;
 183             prp_trans = MIN(n->max_prp_ents, nents) * sizeof(uint64_t);
 184             nvme_addr_read(n, prp2, (void *)prp_list, prp_trans);
 185             while (len != 0) {
 186                 uint64_t prp_ent = le64_to_cpu(prp_list[i]);
 187
 188                 if (i == n->max_prp_ents - 1 && len > n->page_size) {
 189                     if (unlikely(!prp_ent || prp_ent & (n->page_size - 1))) {
 190                         trace_nvme_err_invalid_prplist_ent(prp_ent);
 191                         goto unmap;
 192                     }
 193
 194                     i = 0;
 195                     nents = (len + n->page_size - 1) >> n->page_bits;
 196                     prp_trans = MIN(n->max_prp_ents, nents) * sizeof(uint64_t);
 197                     nvme_addr_read(n, prp_ent, (void *)prp_list,
 198                         prp_trans);
 199                     prp_ent = le64_to_cpu(prp_list[i]);
 200                 }
 201
 202                 if (unlikely(!prp_ent || prp_ent & (n->page_size - 1))) {
 203                     trace_nvme_err_invalid_prplist_ent(prp_ent);
 204                     goto unmap;
 205                 }
 206
 207                 trans_len = MIN(len, n->page_size);
 208                 if (qsg->nsg){
 209                     qemu_sglist_add(qsg, prp_ent, trans_len);
 210                 } else {
 211                     qemu_iovec_add(iov, (void *)&n->cmbuf[prp_ent - n->ctrl_mem.addr], trans_len);
 212                 }
 213                 len -= trans_len;
 214                 i++;
 215             }
 216         } else {
 217             if (unlikely(prp2 & (n->page_size - 1))) {
 218                 trace_nvme_err_invalid_prp2_align(prp2);
 219                 goto unmap;
 220             }
 221             if (qsg->nsg) {
 222                 qemu_sglist_add(qsg, prp2, len);
 223             } else {
 224                 qemu_iovec_add(iov, (void *)&n->cmbuf[prp2 - n->ctrl_mem.addr], trans_len);
 225             }
 226         }
 227     }
 228     return NVME_SUCCESS;
 229
 230  unmap:
 231     qemu_sglist_destroy(qsg);
 232     return NVME_INVALID_FIELD | NVME_DNR;
 233 }
 234
 235 static uint16_t nvme_dma_write_prp(NvmeCtrl *n, uint8_t *ptr, uint32_t len,
 236                                    uint64_t prp1, uint64_t prp2)
 237 {
 238     QEMUSGList qsg;
 239     QEMUIOVector iov;
 240     uint16_t status = NVME_SUCCESS;
 241
 242     if (nvme_map_prp(&qsg, &iov, prp1, prp2, len, n)) {
 243         return NVME_INVALID_FIELD | NVME_DNR;
 244     }
 245     if (qsg.nsg > 0) {
 246         if (dma_buf_write(ptr, len, &qsg)) {
 247             status = NVME_INVALID_FIELD | NVME_DNR;
 248         }
 249         qemu_sglist_destroy(&qsg);
 250     } else {
 251         if (qemu_iovec_to_buf(&iov, 0, ptr, len) != len) {
 252             status = NVME_INVALID_FIELD | NVME_DNR;
 253         }
 254         qemu_iovec_destroy(&iov);
 255     }
 256     return status;
 257 }
 258
 259 static uint16_t nvme_dma_read_prp(NvmeCtrl *n, uint8_t *ptr, uint32_t len,
 260     uint64_t prp1, uint64_t prp2)
 261 {
 262     QEMUSGList qsg;
 263     QEMUIOVector iov;
 264     uint16_t status = NVME_SUCCESS;
 265
 266     trace_nvme_dma_read(prp1, prp2);
 267
 268     if (nvme_map_prp(&qsg, &iov, prp1, prp2, len, n)) {
 269         return NVME_INVALID_FIELD | NVME_DNR;
 270     }
 271     if (qsg.nsg > 0) {
 272         if (unlikely(dma_buf_read(ptr, len, &qsg))) {
 273             trace_nvme_err_invalid_dma();
 274             status = NVME_INVALID_FIELD | NVME_DNR;
 275         }
 276         qemu_sglist_destroy(&qsg);
 277     } else {
 278         if (unlikely(qemu_iovec_from_buf(&iov, 0, ptr, len) != len)) {
 279             trace_nvme_err_invalid_dma();
 280             status = NVME_INVALID_FIELD | NVME_DNR;
 281         }
 282         qemu_iovec_destroy(&iov);
 283     }
 284     return status;
 285 }
 286
 287 static void nvme_post_cqes(void *opaque)
 288 {
 289     NvmeCQueue *cq = opaque;
 290     NvmeCtrl *n = cq->ctrl;
 291     NvmeRequest *req, *next;
 292
 293     QTAILQ_FOREACH_SAFE(req, &cq->req_list, entry, next) {
 294         NvmeSQueue *sq;
 295         hwaddr addr;
 296
 297         if (nvme_cq_full(cq)) {
 298             break;
 299         }
 300
 301         QTAILQ_REMOVE(&cq->req_list, req, entry);
 302         sq = req->sq;
 303         req->cqe.status = cpu_to_le16((req->status << 1) | cq->phase);
 304         req->cqe.sq_id = cpu_to_le16(sq->sqid);
 305         req->cqe.sq_head = cpu_to_le16(sq->head);
 306         addr = cq->dma_addr + cq->tail * n->cqe_size;
 307         nvme_inc_cq_tail(cq);
 308         pci_dma_write(&n->parent_obj, addr, (void *)&req->cqe,
 309             sizeof(req->cqe));
 310         QTAILQ_INSERT_TAIL(&sq->req_list, req, entry);
 311     }
 312     if (cq->tail != cq->head) {
 313         nvme_irq_assert(n, cq);
 314     }
 315 }
 316
 317 static void nvme_enqueue_req_completion(NvmeCQueue *cq, NvmeRequest *req)
 318 {
 319     assert(cq->cqid == req->sq->cqid);
 320     QTAILQ_REMOVE(&req->sq->out_req_list, req, entry);
 321     QTAILQ_INSERT_TAIL(&cq->req_list, req, entry);
 322     timer_mod(cq->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 500);
 323 }
 324
 325 static void nvme_rw_cb(void *opaque, int ret)
 326 {
 327     NvmeRequest *req = opaque;
 328     NvmeSQueue *sq = req->sq;
 329     NvmeCtrl *n = sq->ctrl;
 330     NvmeCQueue *cq = n->cq[sq->cqid];
 331
 332     if (!ret) {
 333         block_acct_done(blk_get_stats(n->conf.blk), &req->acct);
 334         req->status = NVME_SUCCESS;
 335     } else {
 336         block_acct_failed(blk_get_stats(n->conf.blk), &req->acct);
 337         req->status = NVME_INTERNAL_DEV_ERROR;
 338     }
 339     if (req->has_sg) {
 340         qemu_sglist_destroy(&req->qsg);
 341     }
 342     nvme_enqueue_req_completion(cq, req);
 343 }
 344
 345 static uint16_t nvme_flush(NvmeCtrl *n, NvmeNamespace *ns, NvmeCmd *cmd,
 346     NvmeRequest *req)
 347 {
 348     req->has_sg = false;
 349     block_acct_start(blk_get_stats(n->conf.blk), &req->acct, 0,
 350          BLOCK_ACCT_FLUSH);
 351     req->aiocb = blk_aio_flush(n->conf.blk, nvme_rw_cb, req);
 352
 353     return NVME_NO_COMPLETE;
 354 }
 355
 356 static uint16_t nvme_write_zeros(NvmeCtrl *n, NvmeNamespace *ns, NvmeCmd *cmd,
 357     NvmeRequest *req)
 358 {
 359     NvmeRwCmd *rw = (NvmeRwCmd *)cmd;
 360     const uint8_t lba_index = NVME_ID_NS_FLBAS_INDEX(ns->id_ns.flbas);
 361     const uint8_t data_shift = ns->id_ns.lbaf[lba_index].ds;
 362     uint64_t slba = le64_to_cpu(rw->slba);
 363     uint32_t nlb  = le16_to_cpu(rw->nlb) + 1;
 364     uint64_t offset = slba << data_shift;
 365     uint32_t count = nlb << data_shift;
 366
 367     if (unlikely(slba + nlb > ns->id_ns.nsze)) {
 368         trace_nvme_err_invalid_lba_range(slba, nlb, ns->id_ns.nsze);
 369         return NVME_LBA_RANGE | NVME_DNR;
 370     }
 371
 372     req->has_sg = false;
 373     block_acct_start(blk_get_stats(n->conf.blk), &req->acct, 0,
 374                      BLOCK_ACCT_WRITE);
 375     req->aiocb = blk_aio_pwrite_zeroes(n->conf.blk, offset, count,
 376                                         BDRV_REQ_MAY_UNMAP, nvme_rw_cb, req);
 377     return NVME_NO_COMPLETE;
 378 }
 379
 380 static uint16_t nvme_rw(NvmeCtrl *n, NvmeNamespace *ns, NvmeCmd *cmd,
 381     NvmeRequest *req)
 382 {
 383     NvmeRwCmd *rw = (NvmeRwCmd *)cmd;
 384     uint32_t nlb  = le32_to_cpu(rw->nlb) + 1;
 385     uint64_t slba = le64_to_cpu(rw->slba);
 386     uint64_t prp1 = le64_to_cpu(rw->prp1);
 387     uint64_t prp2 = le64_to_cpu(rw->prp2);
 388
 389     uint8_t lba_index  = NVME_ID_NS_FLBAS_INDEX(ns->id_ns.flbas);
 390     uint8_t data_shift = ns->id_ns.lbaf[lba_index].ds;
 391     uint64_t data_size = (uint64_t)nlb << data_shift;
 392     uint64_t data_offset = slba << data_shift;
 393     int is_write = rw->opcode == NVME_CMD_WRITE ? 1 : 0;
 394     enum BlockAcctType acct = is_write ? BLOCK_ACCT_WRITE : BLOCK_ACCT_READ;
 395
 396     trace_nvme_rw(is_write ? "write" : "read", nlb, data_size, slba);
 397
 398     if (unlikely((slba + nlb) > ns->id_ns.nsze)) {
 399         block_acct_invalid(blk_get_stats(n->conf.blk), acct);
 400         trace_nvme_err_invalid_lba_range(slba, nlb, ns->id_ns.nsze);
 401         return NVME_LBA_RANGE | NVME_DNR;
 402     }
 403
 404     if (nvme_map_prp(&req->qsg, &req->iov, prp1, prp2, data_size, n)) {
 405         block_acct_invalid(blk_get_stats(n->conf.blk), acct);
 406         return NVME_INVALID_FIELD | NVME_DNR;
 407     }
 408
 409     dma_acct_start(n->conf.blk, &req->acct, &req->qsg, acct);
 410     if (req->qsg.nsg > 0) {
 411         req->has_sg = true;
 412         req->aiocb = is_write ?
 413             dma_blk_write(n->conf.blk, &req->qsg, data_offset, BDRV_SECTOR_SIZE,
 414                           nvme_rw_cb, req) :
 415             dma_blk_read(n->conf.blk, &req->qsg, data_offset, BDRV_SECTOR_SIZE,
 416                          nvme_rw_cb, req);
 417     } else {
 418         req->has_sg = false;
 419         req->aiocb = is_write ?
 420             blk_aio_pwritev(n->conf.blk, data_offset, &req->iov, 0, nvme_rw_cb,
 421                             req) :
 422             blk_aio_preadv(n->conf.blk, data_offset, &req->iov, 0, nvme_rw_cb,
 423                            req);
 424     }
 425
 426     return NVME_NO_COMPLETE;
 427 }
 428
 429 static uint16_t nvme_io_cmd(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
 430 {
 431     NvmeNamespace *ns;
 432     uint32_t nsid = le32_to_cpu(cmd->nsid);
 433
 434     if (unlikely(nsid == 0 || nsid > n->num_namespaces)) {
 435         trace_nvme_err_invalid_ns(nsid, n->num_namespaces);
 436         return NVME_INVALID_NSID | NVME_DNR;
 437     }
 438
 439     ns = &n->namespaces[nsid - 1];
 440     switch (cmd->opcode) {
 441     case NVME_CMD_FLUSH:
 442         return nvme_flush(n, ns, cmd, req);
 443     case NVME_CMD_WRITE_ZEROS:
 444         return nvme_write_zeros(n, ns, cmd, req);
 445     case NVME_CMD_WRITE:
 446     case NVME_CMD_READ:
 447         return nvme_rw(n, ns, cmd, req);
 448     default:
 449         trace_nvme_err_invalid_opc(cmd->opcode);
 450         return NVME_INVALID_OPCODE | NVME_DNR;
 451     }
 452 }
 453
 454 static void nvme_free_sq(NvmeSQueue *sq, NvmeCtrl *n)
 455 {
 456     n->sq[sq->sqid] = NULL;
 457     timer_del(sq->timer);
 458     timer_free(sq->timer);
 459     g_free(sq->io_req);
 460     if (sq->sqid) {
 461         g_free(sq);
 462     }
 463 }
 464
 465 static uint16_t nvme_del_sq(NvmeCtrl *n, NvmeCmd *cmd)
 466 {
 467     NvmeDeleteQ *c = (NvmeDeleteQ *)cmd;
 468     NvmeRequest *req, *next;
 469     NvmeSQueue *sq;
 470     NvmeCQueue *cq;
 471     uint16_t qid = le16_to_cpu(c->qid);
 472
 473     if (unlikely(!qid || nvme_check_sqid(n, qid))) {
 474         trace_nvme_err_invalid_del_sq(qid);
 475         return NVME_INVALID_QID | NVME_DNR;
 476     }
 477
 478     trace_nvme_del_sq(qid);
 479
 480     sq = n->sq[qid];
 481     while (!QTAILQ_EMPTY(&sq->out_req_list)) {
 482         req = QTAILQ_FIRST(&sq->out_req_list);
 483         assert(req->aiocb);
 484         blk_aio_cancel(req->aiocb);
 485     }
 486     if (!nvme_check_cqid(n, sq->cqid)) {
 487         cq = n->cq[sq->cqid];
 488         QTAILQ_REMOVE(&cq->sq_list, sq, entry);
 489
 490         nvme_post_cqes(cq);
 491         QTAILQ_FOREACH_SAFE(req, &cq->req_list, entry, next) {
 492             if (req->sq == sq) {
 493                 QTAILQ_REMOVE(&cq->req_list, req, entry);
 494                 QTAILQ_INSERT_TAIL(&sq->req_list, req, entry);
 495             }
 496         }
 497     }
 498
 499     nvme_free_sq(sq, n);
 500     return NVME_SUCCESS;
 501 }
 502
 503 static void nvme_init_sq(NvmeSQueue *sq, NvmeCtrl *n, uint64_t dma_addr,
 504     uint16_t sqid, uint16_t cqid, uint16_t size)
 505 {
 506     int i;
 507     NvmeCQueue *cq;
 508
 509     sq->ctrl = n;
 510     sq->dma_addr = dma_addr;
 511     sq->sqid = sqid;
 512     sq->size = size;
 513     sq->cqid = cqid;
 514     sq->head = sq->tail = 0;
 515     sq->io_req = g_new(NvmeRequest, sq->size);
 516
 517     QTAILQ_INIT(&sq->req_list);
 518     QTAILQ_INIT(&sq->out_req_list);
 519     for (i = 0; i < sq->size; i++) {
 520         sq->io_req[i].sq = sq;
 521         QTAILQ_INSERT_TAIL(&(sq->req_list), &sq->io_req[i], entry);
 522     }
 523     sq->timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, nvme_process_sq, sq);
 524
 525     assert(n->cq[cqid]);
 526     cq = n->cq[cqid];
 527     QTAILQ_INSERT_TAIL(&(cq->sq_list), sq, entry);
 528     n->sq[sqid] = sq;
 529 }
 530
 531 static uint16_t nvme_create_sq(NvmeCtrl *n, NvmeCmd *cmd)
 532 {
 533     NvmeSQueue *sq;
 534     NvmeCreateSq *c = (NvmeCreateSq *)cmd;
 535
 536     uint16_t cqid = le16_to_cpu(c->cqid);
 537     uint16_t sqid = le16_to_cpu(c->sqid);
 538     uint16_t qsize = le16_to_cpu(c->qsize);
 539     uint16_t qflags = le16_to_cpu(c->sq_flags);
 540     uint64_t prp1 = le64_to_cpu(c->prp1);
 541
 542     trace_nvme_create_sq(prp1, sqid, cqid, qsize, qflags);
 543
 544     if (unlikely(!cqid || nvme_check_cqid(n, cqid))) {
 545         trace_nvme_err_invalid_create_sq_cqid(cqid);
 546         return NVME_INVALID_CQID | NVME_DNR;
 547     }
 548     if (unlikely(!sqid || !nvme_check_sqid(n, sqid))) {
 549         trace_nvme_err_invalid_create_sq_sqid(sqid);
 550         return NVME_INVALID_QID | NVME_DNR;
 551     }
 552     if (unlikely(!qsize || qsize > NVME_CAP_MQES(n->bar.cap))) {
 553         trace_nvme_err_invalid_create_sq_size(qsize);
 554         return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
 555     }
 556     if (unlikely(!prp1 || prp1 & (n->page_size - 1))) {
 557         trace_nvme_err_invalid_create_sq_addr(prp1);
 558         return NVME_INVALID_FIELD | NVME_DNR;
 559     }
 560     if (unlikely(!(NVME_SQ_FLAGS_PC(qflags)))) {
 561         trace_nvme_err_invalid_create_sq_qflags(NVME_SQ_FLAGS_PC(qflags));
 562         return NVME_INVALID_FIELD | NVME_DNR;
 563     }
 564     sq = g_malloc0(sizeof(*sq));
 565     nvme_init_sq(sq, n, prp1, sqid, cqid, qsize + 1);
 566     return NVME_SUCCESS;
 567 }
 568
 569 static void nvme_free_cq(NvmeCQueue *cq, NvmeCtrl *n)
 570 {
 571     n->cq[cq->cqid] = NULL;
 572     timer_del(cq->timer);
 573     timer_free(cq->timer);
 574     msix_vector_unuse(&n->parent_obj, cq->vector);
 575     if (cq->cqid) {
 576         g_free(cq);
 577     }
 578 }
 579
 580 static uint16_t nvme_del_cq(NvmeCtrl *n, NvmeCmd *cmd)
 581 {
 582     NvmeDeleteQ *c = (NvmeDeleteQ *)cmd;
 583     NvmeCQueue *cq;
 584     uint16_t qid = le16_to_cpu(c->qid);
 585
 586     if (unlikely(!qid || nvme_check_cqid(n, qid))) {
 587         trace_nvme_err_invalid_del_cq_cqid(qid);
 588         return NVME_INVALID_CQID | NVME_DNR;
 589     }
 590
 591     cq = n->cq[qid];
 592     if (unlikely(!QTAILQ_EMPTY(&cq->sq_list))) {
 593         trace_nvme_err_invalid_del_cq_notempty(qid);
 594         return NVME_INVALID_QUEUE_DEL;
 595     }
 596     nvme_irq_deassert(n, cq);
 597     trace_nvme_del_cq(qid);
 598     nvme_free_cq(cq, n);
 599     return NVME_SUCCESS;
 600 }
 601
 602 static void nvme_init_cq(NvmeCQueue *cq, NvmeCtrl *n, uint64_t dma_addr,
 603     uint16_t cqid, uint16_t vector, uint16_t size, uint16_t irq_enabled)
 604 {
 605     cq->ctrl = n;
 606     cq->cqid = cqid;
 607     cq->size = size;
 608     cq->dma_addr = dma_addr;
 609     cq->phase = 1;
 610     cq->irq_enabled = irq_enabled;
 611     cq->vector = vector;
 612     cq->head = cq->tail = 0;
 613     QTAILQ_INIT(&cq->req_list);
 614     QTAILQ_INIT(&cq->sq_list);
 615     msix_vector_use(&n->parent_obj, cq->vector);
 616     n->cq[cqid] = cq;
 617     cq->timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, nvme_post_cqes, cq);
 618 }
 619
 620 static uint16_t nvme_create_cq(NvmeCtrl *n, NvmeCmd *cmd)
 621 {
 622     NvmeCQueue *cq;
 623     NvmeCreateCq *c = (NvmeCreateCq *)cmd;
 624     uint16_t cqid = le16_to_cpu(c->cqid);
 625     uint16_t vector = le16_to_cpu(c->irq_vector);
 626     uint16_t qsize = le16_to_cpu(c->qsize);
 627     uint16_t qflags = le16_to_cpu(c->cq_flags);
 628     uint64_t prp1 = le64_to_cpu(c->prp1);
 629
 630     trace_nvme_create_cq(prp1, cqid, vector, qsize, qflags,
 631                          NVME_CQ_FLAGS_IEN(qflags) != 0);
 632
 633     if (unlikely(!cqid || !nvme_check_cqid(n, cqid))) {
 634         trace_nvme_err_invalid_create_cq_cqid(cqid);
 635         return NVME_INVALID_CQID | NVME_DNR;
 636     }
 637     if (unlikely(!qsize || qsize > NVME_CAP_MQES(n->bar.cap))) {
 638         trace_nvme_err_invalid_create_cq_size(qsize);
 639         return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
 640     }
 641     if (unlikely(!prp1)) {
 642         trace_nvme_err_invalid_create_cq_addr(prp1);
 643         return NVME_INVALID_FIELD | NVME_DNR;
 644     }
 645     if (unlikely(vector > n->num_queues)) {
 646         trace_nvme_err_invalid_create_cq_vector(vector);
 647         return NVME_INVALID_IRQ_VECTOR | NVME_DNR;
 648     }
 649     if (unlikely(!(NVME_CQ_FLAGS_PC(qflags)))) {
 650         trace_nvme_err_invalid_create_cq_qflags(NVME_CQ_FLAGS_PC(qflags));
 651         return NVME_INVALID_FIELD | NVME_DNR;
 652     }
 653
 654     cq = g_malloc0(sizeof(*cq));
 655     nvme_init_cq(cq, n, prp1, cqid, vector, qsize + 1,
 656         NVME_CQ_FLAGS_IEN(qflags));
 657     return NVME_SUCCESS;
 658 }
 659
 660 static uint16_t nvme_identify_ctrl(NvmeCtrl *n, NvmeIdentify *c)
 661 {
 662     uint64_t prp1 = le64_to_cpu(c->prp1);
 663     uint64_t prp2 = le64_to_cpu(c->prp2);
 664
 665     trace_nvme_identify_ctrl();
 666
 667     return nvme_dma_read_prp(n, (uint8_t *)&n->id_ctrl, sizeof(n->id_ctrl),
 668         prp1, prp2);
 669 }
 670
 671 static uint16_t nvme_identify_ns(NvmeCtrl *n, NvmeIdentify *c)
 672 {
 673     NvmeNamespace *ns;
 674     uint32_t nsid = le32_to_cpu(c->nsid);
 675     uint64_t prp1 = le64_to_cpu(c->prp1);
 676     uint64_t prp2 = le64_to_cpu(c->prp2);
 677
 678     trace_nvme_identify_ns(nsid);
 679
 680     if (unlikely(nsid == 0 || nsid > n->num_namespaces)) {
 681         trace_nvme_err_invalid_ns(nsid, n->num_namespaces);
 682         return NVME_INVALID_NSID | NVME_DNR;
 683     }
 684
 685     ns = &n->namespaces[nsid - 1];
 686
 687     return nvme_dma_read_prp(n, (uint8_t *)&ns->id_ns, sizeof(ns->id_ns),
 688         prp1, prp2);
 689 }
 690
 691 static uint16_t nvme_identify_nslist(NvmeCtrl *n, NvmeIdentify *c)
 692 {
 693     static const int data_len = 4 * KiB;
 694     uint32_t min_nsid = le32_to_cpu(c->nsid);
 695     uint64_t prp1 = le64_to_cpu(c->prp1);
 696     uint64_t prp2 = le64_to_cpu(c->prp2);
 697     uint32_t *list;
 698     uint16_t ret;
 699     int i, j = 0;
 700
 701     trace_nvme_identify_nslist(min_nsid);
 702
 703     list = g_malloc0(data_len);
 704     for (i = 0; i < n->num_namespaces; i++) {
 705         if (i < min_nsid) {
 706             continue;
 707         }
 708         list[j++] = cpu_to_le32(i + 1);
 709         if (j == data_len / sizeof(uint32_t)) {
 710             break;
 711         }
 712     }
 713     ret = nvme_dma_read_prp(n, (uint8_t *)list, data_len, prp1, prp2);
 714     g_free(list);
 715     return ret;
 716 }
 717
 718 static uint16_t nvme_identify(NvmeCtrl *n, NvmeCmd *cmd)
 719 {
 720     NvmeIdentify *c = (NvmeIdentify *)cmd;
 721
 722     switch (le32_to_cpu(c->cns)) {
 723     case 0x00:
 724         return nvme_identify_ns(n, c);
 725     case 0x01:
 726         return nvme_identify_ctrl(n, c);
 727     case 0x02:
 728         return nvme_identify_nslist(n, c);
 729     default:
 730         trace_nvme_err_invalid_identify_cns(le32_to_cpu(c->cns));
 731         return NVME_INVALID_FIELD | NVME_DNR;
 732     }
 733 }
 734
 735 static inline void nvme_set_timestamp(NvmeCtrl *n, uint64_t ts)
 736 {
 737     trace_nvme_setfeat_timestamp(ts);
 738
 739     n->host_timestamp = le64_to_cpu(ts);
 740     n->timestamp_set_qemu_clock_ms = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
 741 }
 742
 743 static inline uint64_t nvme_get_timestamp(const NvmeCtrl *n)
 744 {
 745     uint64_t current_time = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
 746     uint64_t elapsed_time = current_time - n->timestamp_set_qemu_clock_ms;
 747
 748     union nvme_timestamp {
 749         struct {
 750             uint64_t timestamp:48;
 751             uint64_t sync:1;
 752             uint64_t origin:3;
 753             uint64_t rsvd1:12;
 754         };
 755         uint64_t all;
 756     };
 757
 758     union nvme_timestamp ts;
 759     ts.all = 0;
 760
 761     /*
 762      * If the sum of the Timestamp value set by the host and the elapsed
 763      * time exceeds 2^48, the value returned should be reduced modulo 2^48.
 764      */
 765     ts.timestamp = (n->host_timestamp + elapsed_time) & 0xffffffffffff;
 766
 767     /* If the host timestamp is non-zero, set the timestamp origin */
 768     ts.origin = n->host_timestamp ? 0x01 : 0x00;
 769
 770     trace_nvme_getfeat_timestamp(ts.all);
 771
 772     return cpu_to_le64(ts.all);
 773 }
 774
 775 static uint16_t nvme_get_feature_timestamp(NvmeCtrl *n, NvmeCmd *cmd)
 776 {
 777     uint64_t prp1 = le64_to_cpu(cmd->prp1);
 778     uint64_t prp2 = le64_to_cpu(cmd->prp2);
 779
 780     uint64_t timestamp = nvme_get_timestamp(n);
 781
 782     return nvme_dma_read_prp(n, (uint8_t *)&timestamp,
 783                                  sizeof(timestamp), prp1, prp2);
 784 }
 785
 786 static uint16_t nvme_get_feature(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
 787 {
 788     uint32_t dw10 = le32_to_cpu(cmd->cdw10);
 789     uint32_t result;
 790
 791     switch (dw10) {
 792     case NVME_VOLATILE_WRITE_CACHE:
 793         result = blk_enable_write_cache(n->conf.blk);
 794         trace_nvme_getfeat_vwcache(result ? "enabled" : "disabled");
 795         break;
 796     case NVME_NUMBER_OF_QUEUES:
 797         result = cpu_to_le32((n->num_queues - 2) | ((n->num_queues - 2) << 16));
 798         trace_nvme_getfeat_numq(result);
 799         break;
 800     case NVME_TIMESTAMP:
 801         return nvme_get_feature_timestamp(n, cmd);
 802         break;
 803     default:
 804         trace_nvme_err_invalid_getfeat(dw10);
 805         return NVME_INVALID_FIELD | NVME_DNR;
 806     }
 807
 808     req->cqe.result = result;
 809     return NVME_SUCCESS;
 810 }
 811
 812 static uint16_t nvme_set_feature_timestamp(NvmeCtrl *n, NvmeCmd *cmd)
 813 {
 814     uint16_t ret;
 815     uint64_t timestamp;
 816     uint64_t prp1 = le64_to_cpu(cmd->prp1);
 817     uint64_t prp2 = le64_to_cpu(cmd->prp2);
 818
 819     ret = nvme_dma_write_prp(n, (uint8_t *)&timestamp,
 820                                 sizeof(timestamp), prp1, prp2);
 821     if (ret != NVME_SUCCESS) {
 822         return ret;
 823     }
 824
 825     nvme_set_timestamp(n, timestamp);
 826
 827     return NVME_SUCCESS;
 828 }
 829
 830 static uint16_t nvme_set_feature(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
 831 {
 832     uint32_t dw10 = le32_to_cpu(cmd->cdw10);
 833     uint32_t dw11 = le32_to_cpu(cmd->cdw11);
 834
 835     switch (dw10) {
 836     case NVME_VOLATILE_WRITE_CACHE:
 837         blk_set_enable_write_cache(n->conf.blk, dw11 & 1);
 838         break;
 839     case NVME_NUMBER_OF_QUEUES:
 840         trace_nvme_setfeat_numq((dw11 & 0xFFFF) + 1,
 841                                 ((dw11 >> 16) & 0xFFFF) + 1,
 842                                 n->num_queues - 1, n->num_queues - 1);
 843         req->cqe.result =
 844             cpu_to_le32((n->num_queues - 2) | ((n->num_queues - 2) << 16));
 845         break;
 846
 847     case NVME_TIMESTAMP:
 848         return nvme_set_feature_timestamp(n, cmd);
 849         break;
 850
 851     default:
 852         trace_nvme_err_invalid_setfeat(dw10);
 853         return NVME_INVALID_FIELD | NVME_DNR;
 854     }
 855     return NVME_SUCCESS;
 856 }
 857
 858 static uint16_t nvme_admin_cmd(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
 859 {
 860     switch (cmd->opcode) {
 861     case NVME_ADM_CMD_DELETE_SQ:
 862         return nvme_del_sq(n, cmd);
 863     case NVME_ADM_CMD_CREATE_SQ:
 864         return nvme_create_sq(n, cmd);
 865     case NVME_ADM_CMD_DELETE_CQ:
 866         return nvme_del_cq(n, cmd);
 867     case NVME_ADM_CMD_CREATE_CQ:
 868         return nvme_create_cq(n, cmd);
 869     case NVME_ADM_CMD_IDENTIFY:
 870         return nvme_identify(n, cmd);
 871     case NVME_ADM_CMD_SET_FEATURES:
 872         return nvme_set_feature(n, cmd, req);
 873     case NVME_ADM_CMD_GET_FEATURES:
 874         return nvme_get_feature(n, cmd, req);
 875     default:
 876         trace_nvme_err_invalid_admin_opc(cmd->opcode);
 877         return NVME_INVALID_OPCODE | NVME_DNR;
 878     }
 879 }
 880
 881 static void nvme_process_sq(void *opaque)
 882 {
 883     NvmeSQueue *sq = opaque;
 884     NvmeCtrl *n = sq->ctrl;
 885     NvmeCQueue *cq = n->cq[sq->cqid];
 886
 887     uint16_t status;
 888     hwaddr addr;
 889     NvmeCmd cmd;
 890     NvmeRequest *req;
 891
 892     while (!(nvme_sq_empty(sq) || QTAILQ_EMPTY(&sq->req_list))) {
 893         addr = sq->dma_addr + sq->head * n->sqe_size;
 894         nvme_addr_read(n, addr, (void *)&cmd, sizeof(cmd));
 895         nvme_inc_sq_head(sq);
 896
 897         req = QTAILQ_FIRST(&sq->req_list);
 898         QTAILQ_REMOVE(&sq->req_list, req, entry);
 899         QTAILQ_INSERT_TAIL(&sq->out_req_list, req, entry);
 900         memset(&req->cqe, 0, sizeof(req->cqe));
 901         req->cqe.cid = cmd.cid;
 902
 903         status = sq->sqid ? nvme_io_cmd(n, &cmd, req) :
 904             nvme_admin_cmd(n, &cmd, req);
 905         if (status != NVME_NO_COMPLETE) {
 906             req->status = status;
 907             nvme_enqueue_req_completion(cq, req);
 908         }
 909     }
 910 }
 911
 912 static void nvme_clear_ctrl(NvmeCtrl *n)
 913 {
 914     int i;
 915
 916     blk_drain(n->conf.blk);
 917
 918     for (i = 0; i < n->num_queues; i++) {
 919         if (n->sq[i] != NULL) {
 920             nvme_free_sq(n->sq[i], n);
 921         }
 922     }
 923     for (i = 0; i < n->num_queues; i++) {
 924         if (n->cq[i] != NULL) {
 925             nvme_free_cq(n->cq[i], n);
 926         }
 927     }
 928
 929     blk_flush(n->conf.blk);
 930     n->bar.cc = 0;
 931 }
 932
 933 static int nvme_start_ctrl(NvmeCtrl *n)
 934 {
 935     uint32_t page_bits = NVME_CC_MPS(n->bar.cc) + 12;
 936     uint32_t page_size = 1 << page_bits;
 937
 938     if (unlikely(n->cq[0])) {
 939         trace_nvme_err_startfail_cq();
 940         return -1;
 941     }
 942     if (unlikely(n->sq[0])) {
 943         trace_nvme_err_startfail_sq();
 944         return -1;
 945     }
 946     if (unlikely(!n->bar.asq)) {
 947         trace_nvme_err_startfail_nbarasq();
 948         return -1;
 949     }
 950     if (unlikely(!n->bar.acq)) {
 951         trace_nvme_err_startfail_nbaracq();
 952         return -1;
 953     }
 954     if (unlikely(n->bar.asq & (page_size - 1))) {
 955         trace_nvme_err_startfail_asq_misaligned(n->bar.asq);
 956         return -1;
 957     }
 958     if (unlikely(n->bar.acq & (page_size - 1))) {
 959         trace_nvme_err_startfail_acq_misaligned(n->bar.acq);
 960         return -1;
 961     }
 962     if (unlikely(NVME_CC_MPS(n->bar.cc) <
 963                  NVME_CAP_MPSMIN(n->bar.cap))) {
 964         trace_nvme_err_startfail_page_too_small(
 965                     NVME_CC_MPS(n->bar.cc),
 966                     NVME_CAP_MPSMIN(n->bar.cap));
 967         return -1;
 968     }
 969     if (unlikely(NVME_CC_MPS(n->bar.cc) >
 970                  NVME_CAP_MPSMAX(n->bar.cap))) {
 971         trace_nvme_err_startfail_page_too_large(
 972                     NVME_CC_MPS(n->bar.cc),
 973                     NVME_CAP_MPSMAX(n->bar.cap));
 974         return -1;
 975     }
 976     if (unlikely(NVME_CC_IOCQES(n->bar.cc) <
 977                  NVME_CTRL_CQES_MIN(n->id_ctrl.cqes))) {
 978         trace_nvme_err_startfail_cqent_too_small(
 979                     NVME_CC_IOCQES(n->bar.cc),
 980                     NVME_CTRL_CQES_MIN(n->bar.cap));
 981         return -1;
 982     }
 983     if (unlikely(NVME_CC_IOCQES(n->bar.cc) >
 984                  NVME_CTRL_CQES_MAX(n->id_ctrl.cqes))) {
 985         trace_nvme_err_startfail_cqent_too_large(
 986                     NVME_CC_IOCQES(n->bar.cc),
 987                     NVME_CTRL_CQES_MAX(n->bar.cap));
 988         return -1;
 989     }
 990     if (unlikely(NVME_CC_IOSQES(n->bar.cc) <
 991                  NVME_CTRL_SQES_MIN(n->id_ctrl.sqes))) {
 992         trace_nvme_err_startfail_sqent_too_small(
 993                     NVME_CC_IOSQES(n->bar.cc),
 994                     NVME_CTRL_SQES_MIN(n->bar.cap));
 995         return -1;
 996     }
 997     if (unlikely(NVME_CC_IOSQES(n->bar.cc) >
 998                  NVME_CTRL_SQES_MAX(n->id_ctrl.sqes))) {
 999         trace_nvme_err_startfail_sqent_too_large(
1000                     NVME_CC_IOSQES(n->bar.cc),
1001                     NVME_CTRL_SQES_MAX(n->bar.cap));
1002         return -1;
1003     }
1004     if (unlikely(!NVME_AQA_ASQS(n->bar.aqa))) {
1005         trace_nvme_err_startfail_asqent_sz_zero();
1006         return -1;
1007     }
1008     if (unlikely(!NVME_AQA_ACQS(n->bar.aqa))) {
1009         trace_nvme_err_startfail_acqent_sz_zero();
1010         return -1;
1011     }
1012
1013     n->page_bits = page_bits;
1014     n->page_size = page_size;
1015     n->max_prp_ents = n->page_size / sizeof(uint64_t);
1016     n->cqe_size = 1 << NVME_CC_IOCQES(n->bar.cc);
1017     n->sqe_size = 1 << NVME_CC_IOSQES(n->bar.cc);
1018     nvme_init_cq(&n->admin_cq, n, n->bar.acq, 0, 0,
1019         NVME_AQA_ACQS(n->bar.aqa) + 1, 1);
1020     nvme_init_sq(&n->admin_sq, n, n->bar.asq, 0, 0,
1021         NVME_AQA_ASQS(n->bar.aqa) + 1);
1022
1023     nvme_set_timestamp(n, 0ULL);
1024
1025     return 0;
1026 }
1027
1028 static void nvme_write_bar(NvmeCtrl *n, hwaddr offset, uint64_t data,
1029     unsigned size)
1030 {
1031     if (unlikely(offset & (sizeof(uint32_t) - 1))) {
1032         NVME_GUEST_ERR(nvme_ub_mmiowr_misaligned32,
1033                        "MMIO write not 32-bit aligned,"
1034                        " offset=0x%"PRIx64"", offset);
1035         /* should be ignored, fall through for now */
1036     }
1037
1038     if (unlikely(size < sizeof(uint32_t))) {
1039         NVME_GUEST_ERR(nvme_ub_mmiowr_toosmall,
1040                        "MMIO write smaller than 32-bits,"
1041                        " offset=0x%"PRIx64", size=%u",
1042                        offset, size);
1043         /* should be ignored, fall through for now */
1044     }
1045
1046     switch (offset) {
1047     case 0xc:   /* INTMS */
1048         if (unlikely(msix_enabled(&(n->parent_obj)))) {
1049             NVME_GUEST_ERR(nvme_ub_mmiowr_intmask_with_msix,
1050                            "undefined access to interrupt mask set"
1051                            " when MSI-X is enabled");
1052             /* should be ignored, fall through for now */
1053         }
1054         n->bar.intms |= data & 0xffffffff;
1055         n->bar.intmc = n->bar.intms;
1056         trace_nvme_mmio_intm_set(data & 0xffffffff,
1057                                  n->bar.intmc);
1058         nvme_irq_check(n);
1059         break;
1060     case 0x10:  /* INTMC */
1061         if (unlikely(msix_enabled(&(n->parent_obj)))) {
1062             NVME_GUEST_ERR(nvme_ub_mmiowr_intmask_with_msix,
1063                            "undefined access to interrupt mask clr"
1064                            " when MSI-X is enabled");
1065             /* should be ignored, fall through for now */
1066         }
1067         n->bar.intms &= ~(data & 0xffffffff);
1068         n->bar.intmc = n->bar.intms;
1069         trace_nvme_mmio_intm_clr(data & 0xffffffff,
1070                                  n->bar.intmc);
1071         nvme_irq_check(n);
1072         break;
1073     case 0x14:  /* CC */
1074         trace_nvme_mmio_cfg(data & 0xffffffff);
1075         /* Windows first sends data, then sends enable bit */
1076         if (!NVME_CC_EN(data) && !NVME_CC_EN(n->bar.cc) &&
1077             !NVME_CC_SHN(data) && !NVME_CC_SHN(n->bar.cc))
1078         {
1079             n->bar.cc = data;
1080         }
1081
1082         if (NVME_CC_EN(data) && !NVME_CC_EN(n->bar.cc)) {
1083             n->bar.cc = data;
1084             if (unlikely(nvme_start_ctrl(n))) {
1085                 trace_nvme_err_startfail();
1086                 n->bar.csts = NVME_CSTS_FAILED;
1087             } else {
1088                 trace_nvme_mmio_start_success();
1089                 n->bar.csts = NVME_CSTS_READY;
1090             }
1091         } else if (!NVME_CC_EN(data) && NVME_CC_EN(n->bar.cc)) {
1092             trace_nvme_mmio_stopped();
1093             nvme_clear_ctrl(n);
1094             n->bar.csts &= ~NVME_CSTS_READY;
1095         }
1096         if (NVME_CC_SHN(data) && !(NVME_CC_SHN(n->bar.cc))) {
1097             trace_nvme_mmio_shutdown_set();
1098             nvme_clear_ctrl(n);
1099             n->bar.cc = data;
1100             n->bar.csts |= NVME_CSTS_SHST_COMPLETE;
1101         } else if (!NVME_CC_SHN(data) && NVME_CC_SHN(n->bar.cc)) {
1102             trace_nvme_mmio_shutdown_cleared();
1103             n->bar.csts &= ~NVME_CSTS_SHST_COMPLETE;
1104             n->bar.cc = data;
1105         }
1106         break;
1107     case 0x1C:  /* CSTS */
1108         if (data & (1 << 4)) {
1109             NVME_GUEST_ERR(nvme_ub_mmiowr_ssreset_w1c_unsupported,
1110                            "attempted to W1C CSTS.NSSRO"
1111                            " but CAP.NSSRS is zero (not supported)");
1112         } else if (data != 0) {
1113             NVME_GUEST_ERR(nvme_ub_mmiowr_ro_csts,
1114                            "attempted to set a read only bit"
1115                            " of controller status");
1116         }
1117         break;
1118     case 0x20:  /* NSSR */
1119         if (data == 0x4E564D65) {
1120             trace_nvme_ub_mmiowr_ssreset_unsupported();
1121         } else {
1122             /* The spec says that writes of other values have no effect */
1123             return;
1124         }
1125         break;
1126     case 0x24:  /* AQA */
1127         n->bar.aqa = data & 0xffffffff;
1128         trace_nvme_mmio_aqattr(data & 0xffffffff);
1129         break;
1130     case 0x28:  /* ASQ */
1131         n->bar.asq = data;
1132         trace_nvme_mmio_asqaddr(data);
1133         break;
1134     case 0x2c:  /* ASQ hi */
1135         n->bar.asq |= data << 32;
1136         trace_nvme_mmio_asqaddr_hi(data, n->bar.asq);
1137         break;
1138     case 0x30:  /* ACQ */
1139         trace_nvme_mmio_acqaddr(data);
1140         n->bar.acq = data;
1141         break;
1142     case 0x34:  /* ACQ hi */
1143         n->bar.acq |= data << 32;
1144         trace_nvme_mmio_acqaddr_hi(data, n->bar.acq);
1145         break;
1146     case 0x38:  /* CMBLOC */
1147         NVME_GUEST_ERR(nvme_ub_mmiowr_cmbloc_reserved,
1148                        "invalid write to reserved CMBLOC"
1149                        " when CMBSZ is zero, ignored");
1150         return;
1151     case 0x3C:  /* CMBSZ */
1152         NVME_GUEST_ERR(nvme_ub_mmiowr_cmbsz_readonly,
1153                        "invalid write to read only CMBSZ, ignored");
1154         return;
1155     case 0xE00: /* PMRCAP */
1156         NVME_GUEST_ERR(nvme_ub_mmiowr_pmrcap_readonly,
1157                        "invalid write to PMRCAP register, ignored");
1158         return;
1159     case 0xE04: /* TODO PMRCTL */
1160         break;
1161     case 0xE08: /* PMRSTS */
1162         NVME_GUEST_ERR(nvme_ub_mmiowr_pmrsts_readonly,
1163                        "invalid write to PMRSTS register, ignored");
1164         return;
1165     case 0xE0C: /* PMREBS */
1166         NVME_GUEST_ERR(nvme_ub_mmiowr_pmrebs_readonly,
1167                        "invalid write to PMREBS register, ignored");
1168         return;
1169     case 0xE10: /* PMRSWTP */
1170         NVME_GUEST_ERR(nvme_ub_mmiowr_pmrswtp_readonly,
1171                        "invalid write to PMRSWTP register, ignored");
1172         return;
1173     case 0xE14: /* TODO PMRMSC */
1174          break;
1175     default:
1176         NVME_GUEST_ERR(nvme_ub_mmiowr_invalid,
1177                        "invalid MMIO write,"
1178                        " offset=0x%"PRIx64", data=%"PRIx64"",
1179                        offset, data);
1180         break;
1181     }
1182 }
1183
1184 static uint64_t nvme_mmio_read(void *opaque, hwaddr addr, unsigned size)
1185 {
1186     NvmeCtrl *n = (NvmeCtrl *)opaque;
1187     uint8_t *ptr = (uint8_t *)&n->bar;
1188     uint64_t val = 0;
1189
1190     if (unlikely(addr & (sizeof(uint32_t) - 1))) {
1191         NVME_GUEST_ERR(nvme_ub_mmiord_misaligned32,
1192                        "MMIO read not 32-bit aligned,"
1193                        " offset=0x%"PRIx64"", addr);
1194         /* should RAZ, fall through for now */
1195     } else if (unlikely(size < sizeof(uint32_t))) {
1196         NVME_GUEST_ERR(nvme_ub_mmiord_toosmall,
1197                        "MMIO read smaller than 32-bits,"
1198                        " offset=0x%"PRIx64"", addr);
1199         /* should RAZ, fall through for now */
1200     }
1201
1202     if (addr < sizeof(n->bar)) {
1203         /*
1204          * When PMRWBM bit 1 is set then read from
1205          * from PMRSTS should ensure prior writes
1206          * made it to persistent media
1207          */
1208         if (addr == 0xE08 &&
1209             (NVME_PMRCAP_PMRWBM(n->bar.pmrcap) & 0x02)) {
1210             qemu_ram_writeback(n->pmrdev->mr.ram_block,
1211                                0, n->pmrdev->size);
1212         }
1213         memcpy(&val, ptr + addr, size);
1214     } else {
1215         NVME_GUEST_ERR(nvme_ub_mmiord_invalid_ofs,
1216                        "MMIO read beyond last register,"
1217                        " offset=0x%"PRIx64", returning 0", addr);
1218     }
1219
1220     return val;
1221 }
1222
1223 static void nvme_process_db(NvmeCtrl *n, hwaddr addr, int val)
1224 {
1225     uint32_t qid;
1226
1227     if (unlikely(addr & ((1 << 2) - 1))) {
1228         NVME_GUEST_ERR(nvme_ub_db_wr_misaligned,
1229                        "doorbell write not 32-bit aligned,"
1230                        " offset=0x%"PRIx64", ignoring", addr);
1231         return;
1232     }
1233
1234     if (((addr - 0x1000) >> 2) & 1) {
1235         /* Completion queue doorbell write */
1236
1237         uint16_t new_head = val & 0xffff;
1238         int start_sqs;
1239         NvmeCQueue *cq;
1240
1241         qid = (addr - (0x1000 + (1 << 2))) >> 3;
1242         if (unlikely(nvme_check_cqid(n, qid))) {
1243             NVME_GUEST_ERR(nvme_ub_db_wr_invalid_cq,
1244                            "completion queue doorbell write"
1245                            " for nonexistent queue,"
1246                            " sqid=%"PRIu32", ignoring", qid);
1247             return;
1248         }
1249
1250         cq = n->cq[qid];
1251         if (unlikely(new_head >= cq->size)) {
1252             NVME_GUEST_ERR(nvme_ub_db_wr_invalid_cqhead,
1253                            "completion queue doorbell write value"
1254                            " beyond queue size, sqid=%"PRIu32","
1255                            " new_head=%"PRIu16", ignoring",
1256                            qid, new_head);
1257             return;
1258         }
1259
1260         start_sqs = nvme_cq_full(cq) ? 1 : 0;
1261         cq->head = new_head;
1262         if (start_sqs) {
1263             NvmeSQueue *sq;
1264             QTAILQ_FOREACH(sq, &cq->sq_list, entry) {
1265                 timer_mod(sq->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 500);
1266             }
1267             timer_mod(cq->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 500);
1268         }
1269
1270         if (cq->tail == cq->head) {
1271             nvme_irq_deassert(n, cq);
1272         }
1273     } else {
1274         /* Submission queue doorbell write */
1275
1276         uint16_t new_tail = val & 0xffff;
1277         NvmeSQueue *sq;
1278
1279         qid = (addr - 0x1000) >> 3;
1280         if (unlikely(nvme_check_sqid(n, qid))) {
1281             NVME_GUEST_ERR(nvme_ub_db_wr_invalid_sq,
1282                            "submission queue doorbell write"
1283                            " for nonexistent queue,"
1284                            " sqid=%"PRIu32", ignoring", qid);
1285             return;
1286         }
1287
1288         sq = n->sq[qid];
1289         if (unlikely(new_tail >= sq->size)) {
1290             NVME_GUEST_ERR(nvme_ub_db_wr_invalid_sqtail,
1291                            "submission queue doorbell write value"
1292                            " beyond queue size, sqid=%"PRIu32","
1293                            " new_tail=%"PRIu16", ignoring",
1294                            qid, new_tail);
1295             return;
1296         }
1297
1298         sq->tail = new_tail;
1299         timer_mod(sq->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 500);
1300     }
1301 }
1302
1303 static void nvme_mmio_write(void *opaque, hwaddr addr, uint64_t data,
1304     unsigned size)
1305 {
1306     NvmeCtrl *n = (NvmeCtrl *)opaque;
1307     if (addr < sizeof(n->bar)) {
1308         nvme_write_bar(n, addr, data, size);
1309     } else if (addr >= 0x1000) {
1310         nvme_process_db(n, addr, data);
1311     }
1312 }
1313
1314 static const MemoryRegionOps nvme_mmio_ops = {
1315     .read = nvme_mmio_read,
1316     .write = nvme_mmio_write,
1317     .endianness = DEVICE_LITTLE_ENDIAN,
1318     .impl = {
1319         .min_access_size = 2,
1320         .max_access_size = 8,
1321     },
1322 };
1323
1324 static void nvme_cmb_write(void *opaque, hwaddr addr, uint64_t data,
1325     unsigned size)
1326 {
1327     NvmeCtrl *n = (NvmeCtrl *)opaque;
1328     stn_le_p(&n->cmbuf[addr], size, data);
1329 }
1330
1331 static uint64_t nvme_cmb_read(void *opaque, hwaddr addr, unsigned size)
1332 {
1333     NvmeCtrl *n = (NvmeCtrl *)opaque;
1334     return ldn_le_p(&n->cmbuf[addr], size);
1335 }
1336
1337 static const MemoryRegionOps nvme_cmb_ops = {
1338     .read = nvme_cmb_read,
1339     .write = nvme_cmb_write,
1340     .endianness = DEVICE_LITTLE_ENDIAN,
1341     .impl = {
1342         .min_access_size = 1,
1343         .max_access_size = 8,
1344     },
1345 };
1346
1347 static void nvme_realize(PCIDevice *pci_dev, Error **errp)
1348 {
1349     NvmeCtrl *n = NVME(pci_dev);
1350     NvmeIdCtrl *id = &n->id_ctrl;
1351
1352     int i;
1353     int64_t bs_size;
1354     uint8_t *pci_conf;
1355
1356     if (!n->num_queues) {
1357         error_setg(errp, "num_queues can't be zero");
1358         return;
1359     }
1360
1361     if (!n->conf.blk) {
1362         error_setg(errp, "drive property not set");
1363         return;
1364     }
1365
1366     bs_size = blk_getlength(n->conf.blk);
1367     if (bs_size < 0) {
1368         error_setg(errp, "could not get backing file size");
1369         return;
1370     }
1371
1372     if (!n->serial) {
1373         error_setg(errp, "serial property not set");
1374         return;
1375     }
1376
1377     if (!n->cmb_size_mb && n->pmrdev) {
1378         if (host_memory_backend_is_mapped(n->pmrdev)) {
1379             char *path = object_get_canonical_path_component(OBJECT(n->pmrdev));
1380             error_setg(errp, "can't use already busy memdev: %s", path);
1381             g_free(path);
1382             return;
1383         }
1384
1385         if (!is_power_of_2(n->pmrdev->size)) {
1386             error_setg(errp, "pmr backend size needs to be power of 2 in size");
1387             return;
1388         }
1389
1390         host_memory_backend_set_mapped(n->pmrdev, true);
1391     }
1392
1393     blkconf_blocksizes(&n->conf);
1394     if (!blkconf_apply_backend_options(&n->conf, blk_is_read_only(n->conf.blk),
1395                                        false, errp)) {
1396         return;
1397     }
1398
1399     pci_conf = pci_dev->config;
1400     pci_conf[PCI_INTERRUPT_PIN] = 1;
1401     pci_config_set_prog_interface(pci_dev->config, 0x2);
1402     pci_config_set_class(pci_dev->config, PCI_CLASS_STORAGE_EXPRESS);
1403     pcie_endpoint_cap_init(pci_dev, 0x80);
1404
1405     n->num_namespaces = 1;
1406     n->reg_size = pow2ceil(0x1004 + 2 * (n->num_queues + 1) * 4);
1407     n->ns_size = bs_size / (uint64_t)n->num_namespaces;
1408
1409     n->namespaces = g_new0(NvmeNamespace, n->num_namespaces);
1410     n->sq = g_new0(NvmeSQueue *, n->num_queues);
1411     n->cq = g_new0(NvmeCQueue *, n->num_queues);
1412
1413     memory_region_init_io(&n->iomem, OBJECT(n), &nvme_mmio_ops, n,
1414                           "nvme", n->reg_size);
1415     pci_register_bar(pci_dev, 0,
1416         PCI_BASE_ADDRESS_SPACE_MEMORY | PCI_BASE_ADDRESS_MEM_TYPE_64,
1417         &n->iomem);
1418     msix_init_exclusive_bar(pci_dev, n->num_queues, 4, NULL);
1419
1420     id->vid = cpu_to_le16(pci_get_word(pci_conf + PCI_VENDOR_ID));
1421     id->ssvid = cpu_to_le16(pci_get_word(pci_conf + PCI_SUBSYSTEM_VENDOR_ID));
1422     strpadcpy((char *)id->mn, sizeof(id->mn), "QEMU NVMe Ctrl", ' ');
1423     strpadcpy((char *)id->fr, sizeof(id->fr), "1.0", ' ');
1424     strpadcpy((char *)id->sn, sizeof(id->sn), n->serial, ' ');
1425     id->rab = 6;
1426     id->ieee[0] = 0x00;
1427     id->ieee[1] = 0x02;
1428     id->ieee[2] = 0xb3;
1429     id->oacs = cpu_to_le16(0);
1430     id->frmw = 7 << 1;
1431     id->lpa = 1 << 0;
1432     id->sqes = (0x6 << 4) | 0x6;
1433     id->cqes = (0x4 << 4) | 0x4;
1434     id->nn = cpu_to_le32(n->num_namespaces);
1435     id->oncs = cpu_to_le16(NVME_ONCS_WRITE_ZEROS | NVME_ONCS_TIMESTAMP);
1436     id->psd[0].mp = cpu_to_le16(0x9c4);
1437     id->psd[0].enlat = cpu_to_le32(0x10);
1438     id->psd[0].exlat = cpu_to_le32(0x4);
1439     if (blk_enable_write_cache(n->conf.blk)) {
1440         id->vwc = 1;
1441     }
1442
1443     n->bar.cap = 0;
1444     NVME_CAP_SET_MQES(n->bar.cap, 0x7ff);
1445     NVME_CAP_SET_CQR(n->bar.cap, 1);
1446     NVME_CAP_SET_TO(n->bar.cap, 0xf);
1447     NVME_CAP_SET_CSS(n->bar.cap, 1);
1448     NVME_CAP_SET_MPSMAX(n->bar.cap, 4);
1449
1450     n->bar.vs = 0x00010200;
1451     n->bar.intmc = n->bar.intms = 0;
1452
1453     if (n->cmb_size_mb) {
1454
1455         NVME_CMBLOC_SET_BIR(n->bar.cmbloc, 2);
1456         NVME_CMBLOC_SET_OFST(n->bar.cmbloc, 0);
1457
1458         NVME_CMBSZ_SET_SQS(n->bar.cmbsz, 1);
1459         NVME_CMBSZ_SET_CQS(n->bar.cmbsz, 0);
1460         NVME_CMBSZ_SET_LISTS(n->bar.cmbsz, 0);
1461         NVME_CMBSZ_SET_RDS(n->bar.cmbsz, 1);
1462         NVME_CMBSZ_SET_WDS(n->bar.cmbsz, 1);
1463         NVME_CMBSZ_SET_SZU(n->bar.cmbsz, 2); /* MBs */
1464         NVME_CMBSZ_SET_SZ(n->bar.cmbsz, n->cmb_size_mb);
1465
1466         n->cmbloc = n->bar.cmbloc;
1467         n->cmbsz = n->bar.cmbsz;
1468
1469         n->cmbuf = g_malloc0(NVME_CMBSZ_GETSIZE(n->bar.cmbsz));
1470         memory_region_init_io(&n->ctrl_mem, OBJECT(n), &nvme_cmb_ops, n,
1471                               "nvme-cmb", NVME_CMBSZ_GETSIZE(n->bar.cmbsz));
1472         pci_register_bar(pci_dev, NVME_CMBLOC_BIR(n->bar.cmbloc),
1473             PCI_BASE_ADDRESS_SPACE_MEMORY | PCI_BASE_ADDRESS_MEM_TYPE_64 |
1474             PCI_BASE_ADDRESS_MEM_PREFETCH, &n->ctrl_mem);
1475
1476     } else if (n->pmrdev) {
1477         /* Controller Capabilities register */
1478         NVME_CAP_SET_PMRS(n->bar.cap, 1);
1479
1480         /* PMR Capabities register */
1481         n->bar.pmrcap = 0;
1482         NVME_PMRCAP_SET_RDS(n->bar.pmrcap, 0);
1483         NVME_PMRCAP_SET_WDS(n->bar.pmrcap, 0);
1484         NVME_PMRCAP_SET_BIR(n->bar.pmrcap, 2);
1485         NVME_PMRCAP_SET_PMRTU(n->bar.pmrcap, 0);
1486         /* Turn on bit 1 support */
1487         NVME_PMRCAP_SET_PMRWBM(n->bar.pmrcap, 0x02);
1488         NVME_PMRCAP_SET_PMRTO(n->bar.pmrcap, 0);
1489         NVME_PMRCAP_SET_CMSS(n->bar.pmrcap, 0);
1490
1491         /* PMR Control register */
1492         n->bar.pmrctl = 0;
1493         NVME_PMRCTL_SET_EN(n->bar.pmrctl, 0);
1494
1495         /* PMR Status register */
1496         n->bar.pmrsts = 0;
1497         NVME_PMRSTS_SET_ERR(n->bar.pmrsts, 0);
1498         NVME_PMRSTS_SET_NRDY(n->bar.pmrsts, 0);
1499         NVME_PMRSTS_SET_HSTS(n->bar.pmrsts, 0);
1500         NVME_PMRSTS_SET_CBAI(n->bar.pmrsts, 0);
1501
1502         /* PMR Elasticity Buffer Size register */
1503         n->bar.pmrebs = 0;
1504         NVME_PMREBS_SET_PMRSZU(n->bar.pmrebs, 0);
1505         NVME_PMREBS_SET_RBB(n->bar.pmrebs, 0);
1506         NVME_PMREBS_SET_PMRWBZ(n->bar.pmrebs, 0);
1507
1508         /* PMR Sustained Write Throughput register */
1509         n->bar.pmrswtp = 0;
1510         NVME_PMRSWTP_SET_PMRSWTU(n->bar.pmrswtp, 0);
1511         NVME_PMRSWTP_SET_PMRSWTV(n->bar.pmrswtp, 0);
1512
1513         /* PMR Memory Space Control register */
1514         n->bar.pmrmsc = 0;
1515         NVME_PMRMSC_SET_CMSE(n->bar.pmrmsc, 0);
1516         NVME_PMRMSC_SET_CBA(n->bar.pmrmsc, 0);
1517
1518         pci_register_bar(pci_dev, NVME_PMRCAP_BIR(n->bar.pmrcap),
1519             PCI_BASE_ADDRESS_SPACE_MEMORY | PCI_BASE_ADDRESS_MEM_TYPE_64 |
1520             PCI_BASE_ADDRESS_MEM_PREFETCH, &n->pmrdev->mr);
1521     }
1522
1523     for (i = 0; i < n->num_namespaces; i++) {
1524         NvmeNamespace *ns = &n->namespaces[i];
1525         NvmeIdNs *id_ns = &ns->id_ns;
1526         id_ns->nsfeat = 0;
1527         id_ns->nlbaf = 0;
1528         id_ns->flbas = 0;
1529         id_ns->mc = 0;
1530         id_ns->dpc = 0;
1531         id_ns->dps = 0;
1532         id_ns->lbaf[0].ds = BDRV_SECTOR_BITS;
1533         id_ns->ncap  = id_ns->nuse = id_ns->nsze =
1534             cpu_to_le64(n->ns_size >>
1535                 id_ns->lbaf[NVME_ID_NS_FLBAS_INDEX(ns->id_ns.flbas)].ds);
1536     }
1537 }
1538
1539 static void nvme_exit(PCIDevice *pci_dev)
1540 {
1541     NvmeCtrl *n = NVME(pci_dev);
1542
1543     nvme_clear_ctrl(n);
1544     g_free(n->namespaces);
1545     g_free(n->cq);
1546     g_free(n->sq);
1547
1548     if (n->cmb_size_mb) {
1549         g_free(n->cmbuf);
1550     }
1551
1552     if (n->pmrdev) {
1553         host_memory_backend_set_mapped(n->pmrdev, false);
1554     }
1555     msix_uninit_exclusive_bar(pci_dev);
1556 }
1557
1558 static Property nvme_props[] = {
1559     DEFINE_BLOCK_PROPERTIES(NvmeCtrl, conf),
1560     DEFINE_PROP_LINK("pmrdev", NvmeCtrl, pmrdev, TYPE_MEMORY_BACKEND,
1561                      HostMemoryBackend *),
1562     DEFINE_PROP_STRING("serial", NvmeCtrl, serial),
1563     DEFINE_PROP_UINT32("cmb_size_mb", NvmeCtrl, cmb_size_mb, 0),
1564     DEFINE_PROP_UINT32("num_queues", NvmeCtrl, num_queues, 64),
1565     DEFINE_PROP_END_OF_LIST(),
1566 };
1567
1568 static const VMStateDescription nvme_vmstate = {
1569     .name = "nvme",
1570     .unmigratable = 1,
1571 };
1572
1573 static void nvme_class_init(ObjectClass *oc, void *data)
1574 {
1575     DeviceClass *dc = DEVICE_CLASS(oc);
1576     PCIDeviceClass *pc = PCI_DEVICE_CLASS(oc);
1577
1578     pc->realize = nvme_realize;
1579     pc->exit = nvme_exit;
1580     pc->class_id = PCI_CLASS_STORAGE_EXPRESS;
1581     pc->vendor_id = PCI_VENDOR_ID_INTEL;
1582     pc->device_id = 0x5845;
1583     pc->revision = 2;
1584
1585     set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
1586     dc->desc = "Non-Volatile Memory Express";
1587     device_class_set_props(dc, nvme_props);
1588     dc->vmsd = &nvme_vmstate;
1589 }
1590
1591 static void nvme_instance_init(Object *obj)
1592 {
1593     NvmeCtrl *s = NVME(obj);
1594
1595     device_add_bootindex_property(obj, &s->conf.bootindex,
1596                                   "bootindex", "/namespace@1,0",
1597                                   DEVICE(obj));
1598 }
1599
1600 static const TypeInfo nvme_info = {
1601     .name          = TYPE_NVME,
1602     .parent        = TYPE_PCI_DEVICE,
1603     .instance_size = sizeof(NvmeCtrl),
1604     .class_init    = nvme_class_init,
1605     .instance_init = nvme_instance_init,
1606     .interfaces = (InterfaceInfo[]) {
1607         { INTERFACE_PCIE_DEVICE },
1608         { }
1609     },
1610 };
1611
1612 static void nvme_register_types(void)
1613 {
1614     type_register_static(&nvme_info);
1615 }
1616
1617 type_init(nvme_register_types)