MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressPassthru.c

   1 /** @file
   2   NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows
   3   NVM Express specification.
   4
   5   Copyright (c) 2013, Intel Corporation. All rights reserved.<BR>
   6   This program and the accompanying materials
   7   are licensed and made available under the terms and conditions of the BSD License
   8   which accompanies this distribution.  The full text of the license may be found at
   9   http://opensource.org/licenses/bsd-license.php.
  10
  11   THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  12   WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  13
  14 **/
  15
  16 #include "NvmExpress.h"
  17
  18 //
  19 // Page size should be set in the Controller Configuration register
  20 // during controller init, and the controller configuration save in
  21 // the controller's private data.  The Max and Min supported page sizes
  22 // for the controller are specified in the Controller Capabilities register.
  23 //
  24
  25 GLOBAL_REMOVE_IF_UNREFERENCED NVM_EXPRESS_PASS_THRU_MODE gNvmExpressPassThruMode = {
  26   0,
  27   NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL | NVM_EXPRESS_PASS_THRU_ATTRIBUTES_CMD_SET_NVME,
  28   sizeof (UINTN),
  29   0x10000,
  30   0,
  31   0
  32 };
  33
  34
  35 /**
  36   Dump the execution status from a given completion queue entry.
  37
  38   @param[in]     Cq               A pointer to the NVME_CQ item.
  39
  40 **/
  41 VOID
  42 NvmeDumpStatus (
  43   IN NVME_CQ             *Cq
  44   )
  45 {
  46   DEBUG ((EFI_D_VERBOSE, "Dump NVMe Completion Entry Status from [0x%x]:\n", Cq));
  47
  48   DEBUG ((EFI_D_VERBOSE, "  SQ Identifier : [0x%x], Phase Tag : [%d], Cmd Identifier : [0x%x]\n", Cq->Sqid, Cq->Pt, Cq->Cid));
  49
  50   DEBUG ((EFI_D_VERBOSE, "  NVMe Cmd Execution Result - "));
  51
  52   switch (Cq->Sct) {
  53     case 0x0:
  54       switch (Cq->Sc) {
  55         case 0x0:
  56           DEBUG ((EFI_D_VERBOSE, "Successful Completion\n"));
  57           break;
  58         case 0x1:
  59           DEBUG ((EFI_D_VERBOSE, "Invalid Command Opcode\n"));
  60           break;
  61         case 0x2:
  62           DEBUG ((EFI_D_VERBOSE, "Invalid Field in Command\n"));
  63           break;
  64         case 0x3:
  65           DEBUG ((EFI_D_VERBOSE, "Command ID Conflict\n"));
  66           break;
  67         case 0x4:
  68           DEBUG ((EFI_D_VERBOSE, "Data Transfer Error\n"));
  69           break;
  70         case 0x5:
  71           DEBUG ((EFI_D_VERBOSE, "Commands Aborted due to Power Loss Notification\n"));
  72           break;
  73         case 0x6:
  74           DEBUG ((EFI_D_VERBOSE, "Internal Device Error\n"));
  75           break;
  76         case 0x7:
  77           DEBUG ((EFI_D_VERBOSE, "Command Abort Requested\n"));
  78           break;
  79         case 0x8:
  80           DEBUG ((EFI_D_VERBOSE, "Command Aborted due to SQ Deletion\n"));
  81           break;
  82         case 0x9:
  83           DEBUG ((EFI_D_VERBOSE, "Command Aborted due to Failed Fused Command\n"));
  84           break;
  85         case 0xA:
  86           DEBUG ((EFI_D_VERBOSE, "Command Aborted due to Missing Fused Command\n"));
  87           break;
  88         case 0xB:
  89           DEBUG ((EFI_D_VERBOSE, "Invalid Namespace or Format\n"));
  90           break;
  91         case 0xC:
  92           DEBUG ((EFI_D_VERBOSE, "Command Sequence Error\n"));
  93           break;
  94         case 0xD:
  95           DEBUG ((EFI_D_VERBOSE, "Invalid SGL Last Segment Descriptor\n"));
  96           break;
  97         case 0xE:
  98           DEBUG ((EFI_D_VERBOSE, "Invalid Number of SGL Descriptors\n"));
  99           break;
 100         case 0xF:
 101           DEBUG ((EFI_D_VERBOSE, "Data SGL Length Invalid\n"));
 102           break;
 103         case 0x10:
 104           DEBUG ((EFI_D_VERBOSE, "Metadata SGL Length Invalid\n"));
 105           break;
 106         case 0x11:
 107           DEBUG ((EFI_D_VERBOSE, "SGL Descriptor Type Invalid\n"));
 108           break;
 109         case 0x80:
 110           DEBUG ((EFI_D_VERBOSE, "LBA Out of Range\n"));
 111           break;
 112         case 0x81:
 113           DEBUG ((EFI_D_VERBOSE, "Capacity Exceeded\n"));
 114           break;
 115         case 0x82:
 116           DEBUG ((EFI_D_VERBOSE, "Namespace Not Ready\n"));
 117           break;
 118         case 0x83:
 119           DEBUG ((EFI_D_VERBOSE, "Reservation Conflict\n"));
 120           break;
 121       }
 122       break;
 123
 124     case 0x1:
 125       switch (Cq->Sc) {
 126         case 0x0:
 127           DEBUG ((EFI_D_VERBOSE, "Completion Queue Invalid\n"));
 128           break;
 129         case 0x1:
 130           DEBUG ((EFI_D_VERBOSE, "Invalid Queue Identifier\n"));
 131           break;
 132         case 0x2:
 133           DEBUG ((EFI_D_VERBOSE, "Maximum Queue Size Exceeded\n"));
 134           break;
 135         case 0x3:
 136           DEBUG ((EFI_D_VERBOSE, "Abort Command Limit Exceeded\n"));
 137           break;
 138         case 0x5:
 139           DEBUG ((EFI_D_VERBOSE, "Asynchronous Event Request Limit Exceeded\n"));
 140           break;
 141         case 0x6:
 142           DEBUG ((EFI_D_VERBOSE, "Invalid Firmware Slot\n"));
 143           break;
 144         case 0x7:
 145           DEBUG ((EFI_D_VERBOSE, "Invalid Firmware Image\n"));
 146           break;
 147         case 0x8:
 148           DEBUG ((EFI_D_VERBOSE, "Invalid Interrupt Vector\n"));
 149           break;
 150         case 0x9:
 151           DEBUG ((EFI_D_VERBOSE, "Invalid Log Page\n"));
 152           break;
 153         case 0xA:
 154           DEBUG ((EFI_D_VERBOSE, "Invalid Format\n"));
 155           break;
 156         case 0xB:
 157           DEBUG ((EFI_D_VERBOSE, "Firmware Application Requires Conventional Reset\n"));
 158           break;
 159         case 0xC:
 160           DEBUG ((EFI_D_VERBOSE, "Invalid Queue Deletion\n"));
 161           break;
 162         case 0xD:
 163           DEBUG ((EFI_D_VERBOSE, "Feature Identifier Not Saveable\n"));
 164           break;
 165         case 0xE:
 166           DEBUG ((EFI_D_VERBOSE, "Feature Not Changeable\n"));
 167           break;
 168         case 0xF:
 169           DEBUG ((EFI_D_VERBOSE, "Feature Not Namespace Specific\n"));
 170           break;
 171         case 0x10:
 172           DEBUG ((EFI_D_VERBOSE, "Firmware Application Requires NVM Subsystem Reset\n"));
 173           break;
 174         case 0x80:
 175           DEBUG ((EFI_D_VERBOSE, "Conflicting Attributes\n"));
 176           break;
 177         case 0x81:
 178           DEBUG ((EFI_D_VERBOSE, "Invalid Protection Information\n"));
 179           break;
 180         case 0x82:
 181           DEBUG ((EFI_D_VERBOSE, "Attempted Write to Read Only Range\n"));
 182           break;
 183       }
 184       break;
 185
 186     case 0x2:
 187       switch (Cq->Sc) {
 188         case 0x80:
 189           DEBUG ((EFI_D_VERBOSE, "Write Fault\n"));
 190           break;
 191         case 0x81:
 192           DEBUG ((EFI_D_VERBOSE, "Unrecovered Read Error\n"));
 193           break;
 194         case 0x82:
 195           DEBUG ((EFI_D_VERBOSE, "End-to-end Guard Check Error\n"));
 196           break;
 197         case 0x83:
 198           DEBUG ((EFI_D_VERBOSE, "End-to-end Application Tag Check Error\n"));
 199           break;
 200         case 0x84:
 201           DEBUG ((EFI_D_VERBOSE, "End-to-end Reference Tag Check Error\n"));
 202           break;
 203         case 0x85:
 204           DEBUG ((EFI_D_VERBOSE, "Compare Failure\n"));
 205           break;
 206         case 0x86:
 207           DEBUG ((EFI_D_VERBOSE, "Access Denied\n"));
 208           break;
 209       }
 210       break;
 211
 212     default:
 213       break;
 214   }
 215 }
 216
 217 /**
 218   Create PRP lists for data transfer which is larger than 2 memory pages.
 219   Note here we calcuate the number of required PRP lists and allocate them at one time.
 220
 221   @param[in]     PciIo               A pointer to the EFI_PCI_IO_PROTOCOL instance.
 222   @param[in]     PhysicalAddr        The physical base address of data buffer.
 223   @param[in]     Pages               The number of pages to be transfered.
 224   @param[out]    PrpListHost         The host base address of PRP lists.
 225   @param[in,out] PrpListNo           The number of PRP List.
 226   @param[out]    Mapping             The mapping value returned from PciIo.Map().
 227
 228   @retval The pointer to the first PRP List of the PRP lists.
 229
 230 **/
 231 VOID*
 232 NvmeCreatePrpList (
 233   IN     EFI_PCI_IO_PROTOCOL          *PciIo,
 234   IN     EFI_PHYSICAL_ADDRESS         PhysicalAddr,
 235   IN     UINTN                        Pages,
 236      OUT VOID                         **PrpListHost,
 237   IN OUT UINTN                        *PrpListNo,
 238      OUT VOID                         **Mapping
 239   )
 240 {
 241   UINTN                       PrpEntryNo;
 242   UINT64                      PrpListBase;
 243   UINTN                       PrpListIndex;
 244   UINTN                       PrpEntryIndex;
 245   UINT64                      Remainder;
 246   EFI_PHYSICAL_ADDRESS        PrpListPhyAddr;
 247   UINTN                       Bytes;
 248   EFI_STATUS                  Status;
 249
 250   //
 251   // The number of Prp Entry in a memory page.
 252   //
 253   PrpEntryNo = EFI_PAGE_SIZE / sizeof (UINT64);
 254
 255   //
 256   // Calculate total PrpList number.
 257   //
 258   *PrpListNo = (UINTN)DivU64x64Remainder ((UINT64)Pages, (UINT64)PrpEntryNo, &Remainder);
 259   if (Remainder != 0) {
 260     *PrpListNo += 1;
 261   }
 262
 263   Status = PciIo->AllocateBuffer (
 264                     PciIo,
 265                     AllocateAnyPages,
 266                     EfiBootServicesData,
 267                     *PrpListNo,
 268                     PrpListHost,
 269                     0
 270                     );
 271
 272   if (EFI_ERROR (Status)) {
 273     return NULL;
 274   }
 275
 276   Bytes = EFI_PAGES_TO_SIZE (*PrpListNo);
 277   Status = PciIo->Map (
 278                     PciIo,
 279                     EfiPciIoOperationBusMasterCommonBuffer,
 280                     *PrpListHost,
 281                     &Bytes,
 282                     &PrpListPhyAddr,
 283                     Mapping
 284                     );
 285
 286   if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (*PrpListNo))) {
 287     DEBUG ((EFI_D_ERROR, "NvmeCreatePrpList: create PrpList failure!\n"));
 288     goto EXIT;
 289   }
 290   //
 291   // Fill all PRP lists except of last one.
 292   //
 293   ZeroMem (*PrpListHost, Bytes);
 294   for (PrpListIndex = 0; PrpListIndex < *PrpListNo - 1; ++PrpListIndex) {
 295     PrpListBase = *(UINT8*)PrpListHost + PrpListIndex * EFI_PAGE_SIZE;
 296
 297     for (PrpEntryIndex = 0; PrpEntryIndex < PrpEntryNo; ++PrpEntryIndex) {
 298       if (PrpEntryIndex != PrpEntryNo - 1) {
 299         //
 300         // Fill all PRP entries except of last one.
 301         //
 302         *((UINT64*)(UINTN)PrpListBase + PrpEntryIndex) = PhysicalAddr;
 303         PhysicalAddr += EFI_PAGE_SIZE;
 304       } else {
 305         //
 306         // Fill last PRP entries with next PRP List pointer.
 307         //
 308         *((UINT64*)(UINTN)PrpListBase + PrpEntryIndex) = PrpListPhyAddr + (PrpListIndex + 1) * EFI_PAGE_SIZE;
 309       }
 310     }
 311   }
 312   //
 313   // Fill last PRP list.
 314   //
 315   PrpListBase = *(UINT64*)PrpListHost + PrpListIndex * EFI_PAGE_SIZE;
 316   for (PrpEntryIndex = 0; PrpEntryIndex < ((Remainder != 0) ? Remainder : PrpEntryNo); ++PrpEntryIndex) {
 317     *((UINT64*)(UINTN)PrpListBase + PrpEntryIndex) = PhysicalAddr;
 318     PhysicalAddr += EFI_PAGE_SIZE;
 319   }
 320
 321   return (VOID*)(UINTN)PrpListPhyAddr;
 322
 323 EXIT:
 324   PciIo->FreeBuffer (PciIo, *PrpListNo, *PrpListHost);
 325   return NULL;
 326 }
 327
 328
 329 /**
 330   Sends an NVM Express Command Packet to an NVM Express controller or namespace. This function supports
 331   both blocking I/O and nonblocking I/O. The blocking I/O functionality is required, and the nonblocking
 332   I/O functionality is optional.
 333
 334   @param[in]     This                A pointer to the NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
 335   @param[in]     NamespaceId         Is a 32 bit Namespace ID to which the Express HCI command packet will be sent.
 336                                      A value of 0 denotes the NVM Express controller, a value of all 0FFh in the namespace
 337                                      ID specifies that the command packet should be sent to all valid namespaces.
 338   @param[in]     NamespaceUuid       Is a 64 bit Namespace UUID to which the Express HCI command packet will be sent.
 339                                      A value of 0 denotes the NVM Express controller, a value of all 0FFh in the namespace
 340                                      UUID specifies that the command packet should be sent to all valid namespaces.
 341   @param[in,out] Packet              A pointer to the NVM Express HCI Command Packet to send to the NVMe namespace specified
 342                                      by NamespaceId.
 343   @param[in]     Event               If nonblocking I/O is not supported then Event is ignored, and blocking I/O is performed.
 344                                      If Event is NULL, then blocking I/O is performed. If Event is not NULL and non blocking I/O
 345                                      is supported, then nonblocking I/O is performed, and Event will be signaled when the NVM
 346                                      Express Command Packet completes.
 347
 348   @retval EFI_SUCCESS                The NVM Express Command Packet was sent by the host. TransferLength bytes were transferred
 349                                      to, or from DataBuffer.
 350   @retval EFI_BAD_BUFFER_SIZE        The NVM Express Command Packet was not executed. The number of bytes that could be transferred
 351                                      is returned in TransferLength.
 352   @retval EFI_NOT_READY              The NVM Express Command Packet could not be sent because the controller is not ready. The caller
 353                                      may retry again later.
 354   @retval EFI_DEVICE_ERROR           A device error occurred while attempting to send the NVM Express Command Packet.
 355   @retval EFI_INVALID_PARAMETER      Namespace, or the contents of NVM_EXPRESS_PASS_THRU_COMMAND_PACKET are invalid. The NVM
 356                                      Express Command Packet was not sent, so no additional status information is available.
 357   @retval EFI_UNSUPPORTED            The command described by the NVM Express Command Packet is not supported by the host adapter.
 358                                      The NVM Express Command Packet was not sent, so no additional status information is available.
 359   @retval EFI_TIMEOUT                A timeout occurred while waiting for the NVM Express Command Packet to execute.
 360
 361 **/
 362 EFI_STATUS
 363 EFIAPI
 364 NvmExpressPassThru (
 365   IN     NVM_EXPRESS_PASS_THRU_PROTOCOL              *This,
 366   IN     UINT32                                      NamespaceId,
 367   IN     UINT64                                      NamespaceUuid,
 368   IN OUT NVM_EXPRESS_PASS_THRU_COMMAND_PACKET        *Packet,
 369   IN     EFI_EVENT                                   Event OPTIONAL
 370   )
 371 {
 372   NVME_CONTROLLER_PRIVATE_DATA  *Private;
 373   EFI_STATUS                    Status;
 374   EFI_PCI_IO_PROTOCOL           *PciIo;
 375   NVME_SQ                       *Sq;
 376   NVME_CQ                       *Cq;
 377   UINT8                         Qid;
 378   UINT32                        Bytes;
 379   UINT16                        Offset;
 380   EFI_EVENT                     TimerEvent;
 381   EFI_PCI_IO_PROTOCOL_OPERATION Flag;
 382   EFI_PHYSICAL_ADDRESS          PhyAddr;
 383   VOID                          *MapData;
 384   VOID                          *MapMeta;
 385   VOID                          *MapPrpList;
 386   UINTN                         MapLength;
 387   UINT64                        *Prp;
 388   VOID                          *PrpListHost;
 389   UINTN                         PrpListNo;
 390
 391   //
 392   // check the data fields in Packet parameter.
 393   //
 394   if ((This == NULL) || (Packet == NULL)) {
 395     return EFI_INVALID_PARAMETER;
 396   }
 397
 398   if ((Packet->NvmeCmd == NULL) || (Packet->NvmeResponse == NULL)) {
 399     return EFI_INVALID_PARAMETER;
 400   }
 401
 402   if (Packet->QueueId != NVME_ADMIN_QUEUE && Packet->QueueId != NVME_IO_QUEUE) {
 403     return EFI_INVALID_PARAMETER;
 404   }
 405
 406   Private     = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
 407   PciIo       = Private->PciIo;
 408   MapData     = NULL;
 409   MapMeta     = NULL;
 410   MapPrpList  = NULL;
 411   PrpListHost = NULL;
 412   PrpListNo   = 0;
 413   Prp         = NULL;
 414   TimerEvent  = NULL;
 415   Status      = EFI_SUCCESS;
 416
 417   Qid = Packet->QueueId;
 418   Sq  = Private->SqBuffer[Qid] + Private->SqTdbl[Qid].Sqt;
 419   Cq  = Private->CqBuffer[Qid] + Private->CqHdbl[Qid].Cqh;
 420
 421   if (Packet->NvmeCmd->Nsid != NamespaceId) {
 422     return EFI_INVALID_PARAMETER;
 423   }
 424
 425   ZeroMem (Sq, sizeof (NVME_SQ));
 426   Sq->Opc  = Packet->NvmeCmd->Cdw0.Opcode;
 427   Sq->Fuse = Packet->NvmeCmd->Cdw0.FusedOperation;
 428   Sq->Cid  = Packet->NvmeCmd->Cdw0.Cid;
 429   Sq->Nsid = Packet->NvmeCmd->Nsid;
 430
 431   //
 432   // Currently we only support PRP for data transfer, SGL is NOT supported.
 433   //
 434   ASSERT ((Sq->Opc & BIT15) == 0);
 435   if ((Sq->Opc & BIT15) != 0) {
 436     DEBUG ((EFI_D_ERROR, "NvmExpressPassThru: doesn't support SGL mechanism\n"));
 437     return EFI_UNSUPPORTED;
 438   }
 439
 440   Sq->Prp[0] = (UINT64)(UINTN)Packet->TransferBuffer;
 441   //
 442   // If the NVMe cmd has data in or out, then mapping the user buffer to the PCI controller specific addresses.
 443   // Note here we don't handle data buffer for CreateIOSubmitionQueue and CreateIOCompletionQueue cmds because
 444   // these two cmds are special which requires their data buffer must support simultaneous access by both the
 445   // processor and a PCI Bus Master. It's caller's responsbility to ensure this.
 446   //
 447   if (((Sq->Opc & (BIT0 | BIT1)) != 0) && (Sq->Opc != NVME_ADMIN_CRIOCQ_OPC) && (Sq->Opc != NVME_ADMIN_CRIOSQ_OPC)) {
 448     if ((Sq->Opc & BIT0) != 0) {
 449       Flag = EfiPciIoOperationBusMasterRead;
 450     } else {
 451       Flag = EfiPciIoOperationBusMasterWrite;
 452     }
 453
 454     MapLength = Packet->TransferLength;
 455     Status = PciIo->Map (
 456                       PciIo,
 457                       Flag,
 458                       Packet->TransferBuffer,
 459                       &MapLength,
 460                       &PhyAddr,
 461                       &MapData
 462                       );
 463     if (EFI_ERROR (Status) || (Packet->TransferLength != MapLength)) {
 464       return EFI_OUT_OF_RESOURCES;
 465     }
 466
 467     Sq->Prp[0] = PhyAddr;
 468     Sq->Prp[1] = 0;
 469
 470     MapLength = Packet->MetadataLength;
 471     if(Packet->MetadataBuffer != NULL) {
 472       MapLength = Packet->MetadataLength;
 473       Status = PciIo->Map (
 474                         PciIo,
 475                         Flag,
 476                         Packet->MetadataBuffer,
 477                         &MapLength,
 478                         &PhyAddr,
 479                         &MapMeta
 480                         );
 481       if (EFI_ERROR (Status) || (Packet->MetadataLength != MapLength)) {
 482         PciIo->Unmap (
 483                  PciIo,
 484                  MapData
 485                  );
 486
 487         return EFI_OUT_OF_RESOURCES;
 488       }
 489       Sq->Mptr = PhyAddr;
 490     }
 491   }
 492   //
 493   // If the buffer size spans more than two memory pages (page size as defined in CC.Mps),
 494   // then build a PRP list in the second PRP submission queue entry.
 495   //
 496   Offset = ((UINT16)Sq->Prp[0]) & (EFI_PAGE_SIZE - 1);
 497   Bytes  = Packet->TransferLength;
 498
 499   if ((Offset + Bytes) > (EFI_PAGE_SIZE * 2)) {
 500     //
 501     // Create PrpList for remaining data buffer.
 502     //
 503     PhyAddr = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
 504     Prp = NvmeCreatePrpList (PciIo, PhyAddr, EFI_SIZE_TO_PAGES(Offset + Bytes) - 1, &PrpListHost, &PrpListNo, &MapPrpList);
 505     if (Prp == NULL) {
 506       goto EXIT;
 507     }
 508
 509     Sq->Prp[1] = (UINT64)(UINTN)Prp;
 510   } else if ((Offset + Bytes) > EFI_PAGE_SIZE) {
 511     Sq->Prp[1] = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
 512   }
 513
 514   if(Packet->NvmeCmd->Flags & CDW10_VALID) {
 515     Sq->Payload.Raw.Cdw10 = Packet->NvmeCmd->Cdw10;
 516   }
 517   if(Packet->NvmeCmd->Flags & CDW11_VALID) {
 518     Sq->Payload.Raw.Cdw11 = Packet->NvmeCmd->Cdw11;
 519   }
 520   if(Packet->NvmeCmd->Flags & CDW12_VALID) {
 521     Sq->Payload.Raw.Cdw12 = Packet->NvmeCmd->Cdw12;
 522   }
 523   if(Packet->NvmeCmd->Flags & CDW13_VALID) {
 524     Sq->Payload.Raw.Cdw13 = Packet->NvmeCmd->Cdw13;
 525   }
 526   if(Packet->NvmeCmd->Flags & CDW14_VALID) {
 527     Sq->Payload.Raw.Cdw14 = Packet->NvmeCmd->Cdw14;
 528   }
 529   if(Packet->NvmeCmd->Flags & CDW15_VALID) {
 530     Sq->Payload.Raw.Cdw15 = Packet->NvmeCmd->Cdw15;
 531   }
 532
 533   //
 534   // Ring the submission queue doorbell.
 535   //
 536   Private->SqTdbl[Qid].Sqt ^= 1;
 537
 538   PciIo->Mem.Write (
 539                PciIo,
 540                EfiPciIoWidthUint32,
 541                NVME_BAR,
 542                NVME_SQTDBL_OFFSET(Qid, Private->Cap.Dstrd),
 543                1,
 544                &Private->SqTdbl[Qid]
 545                );
 546
 547   Status = gBS->CreateEvent (
 548                   EVT_TIMER,
 549                   TPL_CALLBACK,
 550                   NULL,
 551                   NULL,
 552                   &TimerEvent
 553                   );
 554   if (EFI_ERROR (Status)) {
 555     goto EXIT;
 556   }
 557
 558   Status = gBS->SetTimer(TimerEvent, TimerRelative, Packet->CommandTimeout);
 559
 560   if (EFI_ERROR(Status)) {
 561     Packet->ControllerStatus = NVM_EXPRESS_STATUS_CONTROLLER_DEVICE_ERROR;
 562     goto EXIT;
 563   }
 564
 565   //
 566   // Wait for completion queue to get filled in.
 567   //
 568   Status = EFI_TIMEOUT;
 569   Packet->ControllerStatus = NVM_EXPRESS_STATUS_CONTROLLER_TIMEOUT_COMMAND;
 570   while (EFI_ERROR (gBS->CheckEvent (TimerEvent))) {
 571     if (Cq->Pt != Private->Pt[Qid]) {
 572       Status = EFI_SUCCESS;
 573       Packet->ControllerStatus = NVM_EXPRESS_STATUS_CONTROLLER_READY;
 574       break;
 575     }
 576   }
 577
 578   if ((Private->CqHdbl[Qid].Cqh ^= 1) == 0) {
 579     Private->Pt[Qid] ^= 1;
 580   }
 581
 582   //
 583   // Copy the Respose Queue entry for this command to the callers response buffer
 584   //
 585   CopyMem(Packet->NvmeResponse, Cq, sizeof(NVM_EXPRESS_RESPONSE));
 586
 587   //
 588   // Dump every completion entry status for debugging.
 589   //
 590   DEBUG_CODE_BEGIN();
 591     NvmeDumpStatus(Cq);
 592   DEBUG_CODE_END();
 593
 594   PciIo->Mem.Write (
 595                PciIo,
 596                EfiPciIoWidthUint32,
 597                NVME_BAR,
 598                NVME_CQHDBL_OFFSET(Qid, Private->Cap.Dstrd),
 599                1,
 600                &Private->CqHdbl[Qid]
 601                );
 602
 603 EXIT:
 604   if (MapData != NULL) {
 605     PciIo->Unmap (
 606              PciIo,
 607              MapData
 608              );
 609   }
 610
 611   if (MapMeta != NULL) {
 612     PciIo->Unmap (
 613              PciIo,
 614              MapMeta
 615              );
 616   }
 617
 618   if (MapPrpList != NULL) {
 619     PciIo->Unmap (
 620              PciIo,
 621              MapPrpList
 622              );
 623   }
 624
 625   if (Prp != NULL) {
 626     PciIo->FreeBuffer (PciIo, PrpListNo, PrpListHost);
 627   }
 628
 629   if (TimerEvent != NULL) {
 630     gBS->CloseEvent (TimerEvent);
 631   }
 632   return Status;
 633 }
 634
 635 /**
 636   Used to retrieve the list of namespaces defined on an NVM Express controller.
 637
 638   The NVM_EXPRESS_PASS_THRU_PROTOCOL.GetNextNamespace() function retrieves a list of namespaces
 639   defined on an NVM Express controller. If on input a NamespaceID is specified by all 0xFF in the
 640   namespace buffer, then the first namespace defined on the NVM Express controller is returned in
 641   NamespaceID, and a status of EFI_SUCCESS is returned.
 642
 643   If NamespaceId is a Namespace value that was returned on a previous call to GetNextNamespace(),
 644   then the next valid NamespaceId  for an NVM Express SSD namespace on the NVM Express controller
 645   is returned in NamespaceId, and EFI_SUCCESS is returned.
 646
 647   If Namespace array is not a 0xFFFFFFFF and NamespaceId was not returned on a previous call to
 648   GetNextNamespace(), then EFI_INVALID_PARAMETER is returned.
 649
 650   If NamespaceId is the NamespaceId of the last SSD namespace on the NVM Express controller, then
 651   EFI_NOT_FOUND is returned
 652
 653   @param[in]     This           A pointer to the NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
 654   @param[in,out] NamespaceId    On input, a pointer to a legal NamespaceId for an NVM Express
 655                                 namespace present on the NVM Express controller. On output, a
 656                                 pointer to the next NamespaceId of an NVM Express namespace on
 657                                 an NVM Express controller. An input value of 0xFFFFFFFF retrieves
 658                                 the first NamespaceId for an NVM Express namespace present on an
 659                                 NVM Express controller.
 660   @param[out]    NamespaceUuid  On output, the UUID associated with the next namespace, if a UUID
 661                                 is defined for that NamespaceId, otherwise, zero is returned in
 662                                 this parameter. If the caller does not require a UUID, then a NULL
 663                                 pointer may be passed.
 664
 665   @retval EFI_SUCCESS           The NamespaceId of the next Namespace was returned.
 666   @retval EFI_NOT_FOUND         There are no more namespaces defined on this controller.
 667   @retval EFI_INVALID_PARAMETER Namespace array is not a 0xFFFFFFFF and NamespaceId was not returned
 668                                 on a previous call to GetNextNamespace().
 669
 670 **/
 671 EFI_STATUS
 672 EFIAPI
 673 NvmExpressGetNextNamespace (
 674   IN     NVM_EXPRESS_PASS_THRU_PROTOCOL              *This,
 675   IN OUT UINT32                                      *NamespaceId,
 676      OUT UINT64                                      *NamespaceUuid  OPTIONAL
 677   )
 678 {
 679   NVME_CONTROLLER_PRIVATE_DATA     *Private;
 680   NVME_ADMIN_NAMESPACE_DATA        *NamespaceData;
 681   UINT32                           NextNamespaceId;
 682   EFI_STATUS                       Status;
 683
 684   if ((This == NULL) || (NamespaceId == NULL)) {
 685     return EFI_INVALID_PARAMETER;
 686   }
 687
 688   NamespaceData = NULL;
 689   Status        = EFI_NOT_FOUND;
 690
 691   Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
 692   //
 693   // If the NamespaceId input value is 0xFFFFFFFF, then get the first valid namespace ID
 694   //
 695   if (*NamespaceId == 0xFFFFFFFF) {
 696     //
 697     // Start with the first namespace ID
 698     //
 699     NextNamespaceId = 1;
 700     //
 701     // Allocate buffer for Identify Namespace data.
 702     //
 703     NamespaceData = (NVME_ADMIN_NAMESPACE_DATA *)AllocateZeroPool (sizeof (NVME_ADMIN_NAMESPACE_DATA));
 704
 705     if (NamespaceData == NULL) {
 706       return EFI_NOT_FOUND;
 707     }
 708
 709     Status = NvmeIdentifyNamespace (Private, NextNamespaceId, NamespaceData);
 710     if (EFI_ERROR(Status)) {
 711       goto Done;
 712     }
 713
 714     *NamespaceId = NextNamespaceId;
 715     if (NamespaceUuid != NULL) {
 716       *NamespaceUuid = NamespaceData->Eui64;
 717     }
 718   } else {
 719     if (*NamespaceId >= Private->ControllerData->Nn) {
 720       return EFI_INVALID_PARAMETER;
 721     }
 722
 723     NextNamespaceId = *NamespaceId + 1;
 724     //
 725     // Allocate buffer for Identify Namespace data.
 726     //
 727     NamespaceData = (NVME_ADMIN_NAMESPACE_DATA *)AllocateZeroPool (sizeof (NVME_ADMIN_NAMESPACE_DATA));
 728     if (NamespaceData == NULL) {
 729       return EFI_NOT_FOUND;
 730     }
 731
 732     Status = NvmeIdentifyNamespace (Private, NextNamespaceId, NamespaceData);
 733     if (EFI_ERROR(Status)) {
 734       goto Done;
 735     }
 736
 737     *NamespaceId = NextNamespaceId;
 738     if (NamespaceUuid != NULL) {
 739       *NamespaceUuid = NamespaceData->Eui64;
 740     }
 741   }
 742
 743 Done:
 744   if (NamespaceData != NULL) {
 745     FreePool(NamespaceData);
 746   }
 747
 748   return Status;
 749 }
 750
 751 /**
 752   Used to translate a device path node to a Namespace ID and Namespace UUID.
 753
 754   The NVM_EXPRESS_PASS_THRU_PROTOCOL.GetNamwspace() function determines the Namespace ID and Namespace UUID
 755   associated with the NVM Express SSD namespace described by DevicePath. If DevicePath is a device path node type
 756   that the NVM Express Pass Thru driver supports, then the NVM Express Pass Thru driver will attempt to translate
 757   the contents DevicePath into a Namespace ID and UUID. If this translation is successful, then that Namespace ID
 758   and UUID are returned in NamespaceID and NamespaceUUID, and EFI_SUCCESS is returned.
 759
 760   @param[in]  This                A pointer to the NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
 761   @param[in]  DevicePath          A pointer to the device path node that describes an NVM Express namespace on
 762                                   the NVM Express controller.
 763   @param[out] NamespaceId         The NVM Express namespace ID contained in the device path node.
 764   @param[out] NamespaceUuid       The NVM Express namespace contained in the device path node.
 765
 766   @retval EFI_SUCCESS             DevicePath was successfully translated to NamespaceId and NamespaceUuid.
 767   @retval EFI_INVALID_PARAMETER   If DevicePath, NamespaceId, or NamespaceUuid are NULL, then EFI_INVALID_PARAMETER
 768                                   is returned.
 769   @retval EFI_UNSUPPORTED         If DevicePath is not a device path node type that the NVM Express Pass Thru driver
 770                                   supports, then EFI_UNSUPPORTED is returned.
 771   @retval EFI_NOT_FOUND           If DevicePath is a device path node type that the Nvm Express Pass Thru driver
 772                                   supports, but there is not a valid translation from DevicePath to a NamespaceID
 773                                   and NamespaceUuid, then EFI_NOT_FOUND is returned.
 774 **/
 775 EFI_STATUS
 776 EFIAPI
 777 NvmExpressGetNamespace (
 778   IN     NVM_EXPRESS_PASS_THRU_PROTOCOL              *This,
 779   IN     EFI_DEVICE_PATH_PROTOCOL                    *DevicePath,
 780      OUT UINT32                                      *NamespaceId,
 781      OUT UINT64                                      *NamespaceUuid
 782   )
 783 {
 784   NVME_NAMESPACE_DEVICE_PATH       *Node;
 785
 786   if ((This == NULL) || (DevicePath == NULL) || (NamespaceId == NULL) || (NamespaceUuid == NULL)) {
 787     return EFI_INVALID_PARAMETER;
 788   }
 789
 790   if (DevicePath->Type != MESSAGING_DEVICE_PATH) {
 791     return EFI_UNSUPPORTED;
 792   }
 793
 794   Node = (NVME_NAMESPACE_DEVICE_PATH *)DevicePath;
 795
 796   if (DevicePath->SubType == MSG_NVME_NAMESPACE_DP) {
 797     if (DevicePathNodeLength(DevicePath) != sizeof(NVME_NAMESPACE_DEVICE_PATH)) {
 798       return EFI_NOT_FOUND;
 799     }
 800
 801     *NamespaceId   = Node->NamespaceId;
 802     *NamespaceUuid = Node->NamespaceUuid;
 803
 804     return EFI_SUCCESS;
 805   } else {
 806     return EFI_UNSUPPORTED;
 807   }
 808 }
 809
 810 /**
 811   Used to allocate and build a device path node for an NVM Express namespace on an NVM Express controller.
 812
 813   The NVM_EXPRESS_PASS_THRU_PROTOCOL.BuildDevicePath() function allocates and builds a single device
 814   path node for the NVM Express namespace specified by NamespaceId.
 815
 816   If the namespace device specified by NamespaceId is not valid , then EFI_NOT_FOUND is returned.
 817
 818   If DevicePath is NULL, then EFI_INVALID_PARAMETER is returned.
 819
 820   If there are not enough resources to allocate the device path node, then EFI_OUT_OF_RESOURCES is returned.
 821
 822   Otherwise, DevicePath is allocated with the boot service AllocatePool(), the contents of DevicePath are
 823   initialized to describe the NVM Express namespace specified by NamespaceId, and EFI_SUCCESS is returned.
 824
 825   @param[in]     This                A pointer to the NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
 826   @param[in]     NamespaceId         The NVM Express namespace ID  for which a device path node is to be
 827                                      allocated and built. Caller must set the NamespaceId to zero if the
 828                                      device path node will contain a valid UUID.
 829   @param[in]     NamespaceUuid       The NVM Express namespace UUID for which a device path node is to be
 830                                      allocated and built. UUID will only be valid of the Namespace ID is zero.
 831   @param[in,out] DevicePath          A pointer to a single device path node that describes the NVM Express
 832                                      namespace specified by NamespaceId. This function is responsible for
 833                                      allocating the buffer DevicePath with the boot service AllocatePool().
 834                                      It is the caller's responsibility to free DevicePath when the caller
 835                                      is finished with DevicePath.
 836   @retval EFI_SUCCESS                The device path node that describes the NVM Express namespace specified
 837                                      by NamespaceId was allocated and returned in DevicePath.
 838   @retval EFI_NOT_FOUND              The NVM Express namespace specified by NamespaceId does not exist on the
 839                                      NVM Express controller.
 840   @retval EFI_INVALID_PARAMETER      DevicePath is NULL.
 841   @retval EFI_OUT_OF_RESOURCES       There are not enough resources to allocate the DevicePath node.
 842
 843 **/
 844 EFI_STATUS
 845 EFIAPI
 846 NvmExpressBuildDevicePath (
 847   IN     NVM_EXPRESS_PASS_THRU_PROTOCOL              *This,
 848   IN     UINT32                                      NamespaceId,
 849   IN     UINT64                                      NamespaceUuid,
 850   IN OUT EFI_DEVICE_PATH_PROTOCOL                    **DevicePath
 851   )
 852 {
 853   NVME_CONTROLLER_PRIVATE_DATA   *Private;
 854   NVME_NAMESPACE_DEVICE_PATH     *Node;
 855
 856   //
 857   // Validate parameters
 858   //
 859   if ((This == NULL) || (DevicePath == NULL)) {
 860     return EFI_INVALID_PARAMETER;
 861   }
 862
 863   Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
 864
 865   if (NamespaceId == 0) {
 866     return EFI_NOT_FOUND;
 867   }
 868
 869   Node = (NVME_NAMESPACE_DEVICE_PATH *)AllocateZeroPool (sizeof (NVME_NAMESPACE_DEVICE_PATH));
 870
 871   if (Node == NULL) {
 872     return EFI_OUT_OF_RESOURCES;
 873   }
 874
 875   Node->Header.Type    = MESSAGING_DEVICE_PATH;
 876   Node->Header.SubType = MSG_NVME_NAMESPACE_DP;
 877   SetDevicePathNodeLength (&Node->Header, sizeof (NVME_NAMESPACE_DEVICE_PATH));
 878   Node->NamespaceId    = NamespaceId;
 879   Node->NamespaceUuid  = NamespaceUuid;
 880
 881   *DevicePath = (EFI_DEVICE_PATH_PROTOCOL *)Node;
 882   return EFI_SUCCESS;
 883 }
 884