MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressPassthru.c

   1 /** @file
   2   NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows
   3   NVM Express specification.
   4
   5   Copyright (c) 2013, Intel Corporation. All rights reserved.<BR>
   6   This program and the accompanying materials
   7   are licensed and made available under the terms and conditions of the BSD License
   8   which accompanies this distribution.  The full text of the license may be found at
   9   http://opensource.org/licenses/bsd-license.php.
  10
  11   THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  12   WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  13
  14 **/
  15
  16 #include "NvmExpress.h"
  17
  18 //
  19 // Page size should be set in the Controller Configuration register
  20 // during controller init, and the controller configuration save in
  21 // the controller's private data.  The Max and Min supported page sizes
  22 // for the controller are specified in the Controller Capabilities register.
  23 //
  24
  25 GLOBAL_REMOVE_IF_UNREFERENCED NVM_EXPRESS_PASS_THRU_MODE gNvmExpressPassThruMode = {
  26   0,
  27   NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL | NVM_EXPRESS_PASS_THRU_ATTRIBUTES_CMD_SET_NVME,
  28   sizeof (UINTN),
  29   0x10000,
  30   0,
  31   0
  32 };
  33
  34
  35 /**
  36   Dump the execution status from a given completion queue entry.
  37
  38   @param[in]     Cq               A pointer to the NVME_CQ item.
  39
  40 **/
  41 VOID
  42 NvmeDumpStatus (
  43   IN NVME_CQ             *Cq
  44   )
  45 {
  46   DEBUG ((EFI_D_VERBOSE, "Dump NVMe Completion Entry Status from [0x%x]:\n", Cq));
  47
  48   DEBUG ((EFI_D_VERBOSE, "  SQ Identifier : [0x%x], Phase Tag : [%d], Cmd Identifier : [0x%x]\n", Cq->Sqid, Cq->Pt, Cq->Cid));
  49
  50   DEBUG ((EFI_D_VERBOSE, "  NVMe Cmd Execution Result - "));
  51
  52   switch (Cq->Sct) {
  53     case 0x0:
  54       switch (Cq->Sc) {
  55         case 0x0:
  56           DEBUG ((EFI_D_VERBOSE, "Successful Completion\n"));
  57           break;
  58         case 0x1:
  59           DEBUG ((EFI_D_VERBOSE, "Invalid Command Opcode\n"));
  60           break;
  61         case 0x2:
  62           DEBUG ((EFI_D_VERBOSE, "Invalid Field in Command\n"));
  63           break;
  64         case 0x3:
  65           DEBUG ((EFI_D_VERBOSE, "Command ID Conflict\n"));
  66           break;
  67         case 0x4:
  68           DEBUG ((EFI_D_VERBOSE, "Data Transfer Error\n"));
  69           break;
  70         case 0x5:
  71           DEBUG ((EFI_D_VERBOSE, "Commands Aborted due to Power Loss Notification\n"));
  72           break;
  73         case 0x6:
  74           DEBUG ((EFI_D_VERBOSE, "Internal Device Error\n"));
  75           break;
  76         case 0x7:
  77           DEBUG ((EFI_D_VERBOSE, "Command Abort Requested\n"));
  78           break;
  79         case 0x8:
  80           DEBUG ((EFI_D_VERBOSE, "Command Aborted due to SQ Deletion\n"));
  81           break;
  82         case 0x9:
  83           DEBUG ((EFI_D_VERBOSE, "Command Aborted due to Failed Fused Command\n"));
  84           break;
  85         case 0xA:
  86           DEBUG ((EFI_D_VERBOSE, "Command Aborted due to Missing Fused Command\n"));
  87           break;
  88         case 0xB:
  89           DEBUG ((EFI_D_VERBOSE, "Invalid Namespace or Format\n"));
  90           break;
  91         case 0xC:
  92           DEBUG ((EFI_D_VERBOSE, "Command Sequence Error\n"));
  93           break;
  94         case 0xD:
  95           DEBUG ((EFI_D_VERBOSE, "Invalid SGL Last Segment Descriptor\n"));
  96           break;
  97         case 0xE:
  98           DEBUG ((EFI_D_VERBOSE, "Invalid Number of SGL Descriptors\n"));
  99           break;
 100         case 0xF:
 101           DEBUG ((EFI_D_VERBOSE, "Data SGL Length Invalid\n"));
 102           break;
 103         case 0x10:
 104           DEBUG ((EFI_D_VERBOSE, "Metadata SGL Length Invalid\n"));
 105           break;
 106         case 0x11:
 107           DEBUG ((EFI_D_VERBOSE, "SGL Descriptor Type Invalid\n"));
 108           break;
 109         case 0x80:
 110           DEBUG ((EFI_D_VERBOSE, "LBA Out of Range\n"));
 111           break;
 112         case 0x81:
 113           DEBUG ((EFI_D_VERBOSE, "Capacity Exceeded\n"));
 114           break;
 115         case 0x82:
 116           DEBUG ((EFI_D_VERBOSE, "Namespace Not Ready\n"));
 117           break;
 118         case 0x83:
 119           DEBUG ((EFI_D_VERBOSE, "Reservation Conflict\n"));
 120           break;
 121       }
 122       break;
 123
 124     case 0x1:
 125       switch (Cq->Sc) {
 126         case 0x0:
 127           DEBUG ((EFI_D_VERBOSE, "Completion Queue Invalid\n"));
 128           break;
 129         case 0x1:
 130           DEBUG ((EFI_D_VERBOSE, "Invalid Queue Identifier\n"));
 131           break;
 132         case 0x2:
 133           DEBUG ((EFI_D_VERBOSE, "Maximum Queue Size Exceeded\n"));
 134           break;
 135         case 0x3:
 136           DEBUG ((EFI_D_VERBOSE, "Abort Command Limit Exceeded\n"));
 137           break;
 138         case 0x5:
 139           DEBUG ((EFI_D_VERBOSE, "Asynchronous Event Request Limit Exceeded\n"));
 140           break;
 141         case 0x6:
 142           DEBUG ((EFI_D_VERBOSE, "Invalid Firmware Slot\n"));
 143           break;
 144         case 0x7:
 145           DEBUG ((EFI_D_VERBOSE, "Invalid Firmware Image\n"));
 146           break;
 147         case 0x8:
 148           DEBUG ((EFI_D_VERBOSE, "Invalid Interrupt Vector\n"));
 149           break;
 150         case 0x9:
 151           DEBUG ((EFI_D_VERBOSE, "Invalid Log Page\n"));
 152           break;
 153         case 0xA:
 154           DEBUG ((EFI_D_VERBOSE, "Invalid Format\n"));
 155           break;
 156         case 0xB:
 157           DEBUG ((EFI_D_VERBOSE, "Firmware Application Requires Conventional Reset\n"));
 158           break;
 159         case 0xC:
 160           DEBUG ((EFI_D_VERBOSE, "Invalid Queue Deletion\n"));
 161           break;
 162         case 0xD:
 163           DEBUG ((EFI_D_VERBOSE, "Feature Identifier Not Saveable\n"));
 164           break;
 165         case 0xE:
 166           DEBUG ((EFI_D_VERBOSE, "Feature Not Changeable\n"));
 167           break;
 168         case 0xF:
 169           DEBUG ((EFI_D_VERBOSE, "Feature Not Namespace Specific\n"));
 170           break;
 171         case 0x10:
 172           DEBUG ((EFI_D_VERBOSE, "Firmware Application Requires NVM Subsystem Reset\n"));
 173           break;
 174         case 0x80:
 175           DEBUG ((EFI_D_VERBOSE, "Conflicting Attributes\n"));
 176           break;
 177         case 0x81:
 178           DEBUG ((EFI_D_VERBOSE, "Invalid Protection Information\n"));
 179           break;
 180         case 0x82:
 181           DEBUG ((EFI_D_VERBOSE, "Attempted Write to Read Only Range\n"));
 182           break;
 183       }
 184       break;
 185
 186     case 0x2:
 187       switch (Cq->Sc) {
 188         case 0x80:
 189           DEBUG ((EFI_D_VERBOSE, "Write Fault\n"));
 190           break;
 191         case 0x81:
 192           DEBUG ((EFI_D_VERBOSE, "Unrecovered Read Error\n"));
 193           break;
 194         case 0x82:
 195           DEBUG ((EFI_D_VERBOSE, "End-to-end Guard Check Error\n"));
 196           break;
 197         case 0x83:
 198           DEBUG ((EFI_D_VERBOSE, "End-to-end Application Tag Check Error\n"));
 199           break;
 200         case 0x84:
 201           DEBUG ((EFI_D_VERBOSE, "End-to-end Reference Tag Check Error\n"));
 202           break;
 203         case 0x85:
 204           DEBUG ((EFI_D_VERBOSE, "Compare Failure\n"));
 205           break;
 206         case 0x86:
 207           DEBUG ((EFI_D_VERBOSE, "Access Denied\n"));
 208           break;
 209       }
 210       break;
 211
 212     default:
 213       break;
 214   }
 215 }
 216
 217 /**
 218   Create PRP lists for data transfer which is larger than 2 memory pages.
 219   Note here we calcuate the number of required PRP lists and allocate them at one time.
 220
 221   @param[in]     PciIo               A pointer to the EFI_PCI_IO_PROTOCOL instance.
 222   @param[in]     PhysicalAddr        The physical base address of data buffer.
 223   @param[in]     Pages               The number of pages to be transfered.
 224   @param[out]    PrpListHost         The host base address of PRP lists.
 225   @param[in,out] PrpListNo           The number of PRP List.
 226   @param[out]    Mapping             The mapping value returned from PciIo.Map().
 227
 228   @retval The pointer to the first PRP List of the PRP lists.
 229
 230 **/
 231 VOID*
 232 NvmeCreatePrpList (
 233   IN     EFI_PCI_IO_PROTOCOL          *PciIo,
 234   IN     EFI_PHYSICAL_ADDRESS         PhysicalAddr,
 235   IN     UINTN                        Pages,
 236      OUT VOID                         **PrpListHost,
 237   IN OUT UINTN                        *PrpListNo,
 238      OUT VOID                         **Mapping
 239   )
 240 {
 241   UINTN                       PrpEntryNo;
 242   UINT64                      PrpListBase;
 243   UINTN                       PrpListIndex;
 244   UINTN                       PrpEntryIndex;
 245   UINT64                      Remainder;
 246   EFI_PHYSICAL_ADDRESS        PrpListPhyAddr;
 247   UINTN                       Bytes;
 248   EFI_STATUS                  Status;
 249
 250   //
 251   // The number of Prp Entry in a memory page.
 252   //
 253   PrpEntryNo = EFI_PAGE_SIZE / sizeof (UINT64);
 254
 255   //
 256   // Calculate total PrpList number.
 257   //
 258   *PrpListNo = (UINTN)DivU64x64Remainder ((UINT64)Pages, (UINT64)PrpEntryNo, &Remainder);
 259   if (Remainder != 0) {
 260     *PrpListNo += 1;
 261   }
 262
 263   Status = PciIo->AllocateBuffer (
 264                     PciIo,
 265                     AllocateAnyPages,
 266                     EfiBootServicesData,
 267                     *PrpListNo,
 268                     PrpListHost,
 269                     0
 270                     );
 271
 272   if (EFI_ERROR (Status)) {
 273     return NULL;
 274   }
 275
 276   Bytes = EFI_PAGES_TO_SIZE (*PrpListNo);
 277   Status = PciIo->Map (
 278                     PciIo,
 279                     EfiPciIoOperationBusMasterCommonBuffer,
 280                     *PrpListHost,
 281                     &Bytes,
 282                     &PrpListPhyAddr,
 283                     Mapping
 284                     );
 285
 286   if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (*PrpListNo))) {
 287     DEBUG ((EFI_D_ERROR, "NvmeCreatePrpList: create PrpList failure!\n"));
 288     goto EXIT;
 289   }
 290   //
 291   // Fill all PRP lists except of last one.
 292   //
 293   ZeroMem (*PrpListHost, Bytes);
 294   for (PrpListIndex = 0; PrpListIndex < *PrpListNo - 1; ++PrpListIndex) {
 295     PrpListBase = *(UINT8*)PrpListHost + PrpListIndex * EFI_PAGE_SIZE;
 296
 297     for (PrpEntryIndex = 0; PrpEntryIndex < PrpEntryNo; ++PrpEntryIndex) {
 298       if (PrpEntryIndex != PrpEntryNo - 1) {
 299         //
 300         // Fill all PRP entries except of last one.
 301         //
 302         *((UINT64*)(UINTN)PrpListBase + PrpEntryIndex) = PhysicalAddr;
 303         PhysicalAddr += EFI_PAGE_SIZE;
 304       } else {
 305         //
 306         // Fill last PRP entries with next PRP List pointer.
 307         //
 308         *((UINT64*)(UINTN)PrpListBase + PrpEntryIndex) = PrpListPhyAddr + (PrpListIndex + 1) * EFI_PAGE_SIZE;
 309       }
 310     }
 311   }
 312   //
 313   // Fill last PRP list.
 314   //
 315   PrpListBase = *(UINT64*)PrpListHost + PrpListIndex * EFI_PAGE_SIZE;
 316   for (PrpEntryIndex = 0; PrpEntryIndex < ((Remainder != 0) ? Remainder : PrpEntryNo); ++PrpEntryIndex) {
 317     *((UINT64*)(UINTN)PrpListBase + PrpEntryIndex) = PhysicalAddr;
 318     PhysicalAddr += EFI_PAGE_SIZE;
 319   }
 320
 321   return (VOID*)(UINTN)PrpListPhyAddr;
 322
 323 EXIT:
 324   PciIo->FreeBuffer (PciIo, *PrpListNo, *PrpListHost);
 325   return NULL;
 326 }
 327
 328
 329 /**
 330   Sends an NVM Express Command Packet to an NVM Express controller or namespace. This function supports
 331   both blocking I/O and nonblocking I/O. The blocking I/O functionality is required, and the nonblocking
 332   I/O functionality is optional.
 333
 334   @param[in]     This                A pointer to the NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
 335   @param[in]     NamespaceId         Is a 32 bit Namespace ID to which the Express HCI command packet will be sent.
 336                                      A value of 0 denotes the NVM Express controller, a value of all 0FFh in the namespace
 337                                      ID specifies that the command packet should be sent to all valid namespaces.
 338   @param[in]     NamespaceUuid       Is a 64 bit Namespace UUID to which the Express HCI command packet will be sent.
 339                                      A value of 0 denotes the NVM Express controller, a value of all 0FFh in the namespace
 340                                      UUID specifies that the command packet should be sent to all valid namespaces.
 341   @param[in,out] Packet              A pointer to the NVM Express HCI Command Packet to send to the NVMe namespace specified
 342                                      by NamespaceId.
 343   @param[in]     Event               If nonblocking I/O is not supported then Event is ignored, and blocking I/O is performed.
 344                                      If Event is NULL, then blocking I/O is performed. If Event is not NULL and non blocking I/O
 345                                      is supported, then nonblocking I/O is performed, and Event will be signaled when the NVM
 346                                      Express Command Packet completes.
 347
 348   @retval EFI_SUCCESS                The NVM Express Command Packet was sent by the host. TransferLength bytes were transferred
 349                                      to, or from DataBuffer.
 350   @retval EFI_BAD_BUFFER_SIZE        The NVM Express Command Packet was not executed. The number of bytes that could be transferred
 351                                      is returned in TransferLength.
 352   @retval EFI_NOT_READY              The NVM Express Command Packet could not be sent because the controller is not ready. The caller
 353                                      may retry again later.
 354   @retval EFI_DEVICE_ERROR           A device error occurred while attempting to send the NVM Express Command Packet.
 355   @retval EFI_INVALID_PARAMETER      Namespace, or the contents of NVM_EXPRESS_PASS_THRU_COMMAND_PACKET are invalid. The NVM
 356                                      Express Command Packet was not sent, so no additional status information is available.
 357   @retval EFI_UNSUPPORTED            The command described by the NVM Express Command Packet is not supported by the host adapter.
 358                                      The NVM Express Command Packet was not sent, so no additional status information is available.
 359   @retval EFI_TIMEOUT                A timeout occurred while waiting for the NVM Express Command Packet to execute.
 360
 361 **/
 362 EFI_STATUS
 363 EFIAPI
 364 NvmExpressPassThru (
 365   IN     NVM_EXPRESS_PASS_THRU_PROTOCOL              *This,
 366   IN     UINT32                                      NamespaceId,
 367   IN     UINT64                                      NamespaceUuid,
 368   IN OUT NVM_EXPRESS_PASS_THRU_COMMAND_PACKET        *Packet,
 369   IN     EFI_EVENT                                   Event OPTIONAL
 370   )
 371 {
 372   NVME_CONTROLLER_PRIVATE_DATA  *Private;
 373   EFI_STATUS                    Status;
 374   EFI_PCI_IO_PROTOCOL           *PciIo;
 375   NVME_SQ                       *Sq;
 376   NVME_CQ                       *Cq;
 377   UINT8                         Qid;
 378   UINT32                        Bytes;
 379   UINT16                        Offset;
 380   EFI_EVENT                     TimerEvent;
 381   EFI_PCI_IO_PROTOCOL_OPERATION Flag;
 382   EFI_PHYSICAL_ADDRESS          PhyAddr;
 383   VOID                          *MapData;
 384   VOID                          *MapMeta;
 385   VOID                          *MapPrpList;
 386   UINTN                         MapLength;
 387   UINT64                        *Prp;
 388   VOID                          *PrpListHost;
 389   UINTN                         PrpListNo;
 390   UINT32                        Data;
 391
 392   //
 393   // check the data fields in Packet parameter.
 394   //
 395   if ((This == NULL) || (Packet == NULL)) {
 396     return EFI_INVALID_PARAMETER;
 397   }
 398
 399   if ((Packet->NvmeCmd == NULL) || (Packet->NvmeResponse == NULL)) {
 400     return EFI_INVALID_PARAMETER;
 401   }
 402
 403   if (Packet->QueueId != NVME_ADMIN_QUEUE && Packet->QueueId != NVME_IO_QUEUE) {
 404     return EFI_INVALID_PARAMETER;
 405   }
 406
 407   Private     = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
 408   PciIo       = Private->PciIo;
 409   MapData     = NULL;
 410   MapMeta     = NULL;
 411   MapPrpList  = NULL;
 412   PrpListHost = NULL;
 413   PrpListNo   = 0;
 414   Prp         = NULL;
 415   TimerEvent  = NULL;
 416   Status      = EFI_SUCCESS;
 417
 418   Qid = Packet->QueueId;
 419   Sq  = Private->SqBuffer[Qid] + Private->SqTdbl[Qid].Sqt;
 420   Cq  = Private->CqBuffer[Qid] + Private->CqHdbl[Qid].Cqh;
 421
 422   if (Packet->NvmeCmd->Nsid != NamespaceId) {
 423     return EFI_INVALID_PARAMETER;
 424   }
 425
 426   ZeroMem (Sq, sizeof (NVME_SQ));
 427   Sq->Opc  = Packet->NvmeCmd->Cdw0.Opcode;
 428   Sq->Fuse = Packet->NvmeCmd->Cdw0.FusedOperation;
 429   Sq->Cid  = Packet->NvmeCmd->Cdw0.Cid;
 430   Sq->Nsid = Packet->NvmeCmd->Nsid;
 431
 432   //
 433   // Currently we only support PRP for data transfer, SGL is NOT supported.
 434   //
 435   ASSERT (Sq->Psdt == 0);
 436   if (Sq->Psdt != 0) {
 437     DEBUG ((EFI_D_ERROR, "NvmExpressPassThru: doesn't support SGL mechanism\n"));
 438     return EFI_UNSUPPORTED;
 439   }
 440
 441   Sq->Prp[0] = (UINT64)(UINTN)Packet->TransferBuffer;
 442   //
 443   // If the NVMe cmd has data in or out, then mapping the user buffer to the PCI controller specific addresses.
 444   // Note here we don't handle data buffer for CreateIOSubmitionQueue and CreateIOCompletionQueue cmds because
 445   // these two cmds are special which requires their data buffer must support simultaneous access by both the
 446   // processor and a PCI Bus Master. It's caller's responsbility to ensure this.
 447   //
 448   if (((Sq->Opc & (BIT0 | BIT1)) != 0) && (Sq->Opc != NVME_ADMIN_CRIOCQ_OPC) && (Sq->Opc != NVME_ADMIN_CRIOSQ_OPC)) {
 449     if ((Sq->Opc & BIT0) != 0) {
 450       Flag = EfiPciIoOperationBusMasterRead;
 451     } else {
 452       Flag = EfiPciIoOperationBusMasterWrite;
 453     }
 454
 455     MapLength = Packet->TransferLength;
 456     Status = PciIo->Map (
 457                       PciIo,
 458                       Flag,
 459                       Packet->TransferBuffer,
 460                       &MapLength,
 461                       &PhyAddr,
 462                       &MapData
 463                       );
 464     if (EFI_ERROR (Status) || (Packet->TransferLength != MapLength)) {
 465       return EFI_OUT_OF_RESOURCES;
 466     }
 467
 468     Sq->Prp[0] = PhyAddr;
 469     Sq->Prp[1] = 0;
 470
 471     MapLength = Packet->MetadataLength;
 472     if(Packet->MetadataBuffer != NULL) {
 473       MapLength = Packet->MetadataLength;
 474       Status = PciIo->Map (
 475                         PciIo,
 476                         Flag,
 477                         Packet->MetadataBuffer,
 478                         &MapLength,
 479                         &PhyAddr,
 480                         &MapMeta
 481                         );
 482       if (EFI_ERROR (Status) || (Packet->MetadataLength != MapLength)) {
 483         PciIo->Unmap (
 484                  PciIo,
 485                  MapData
 486                  );
 487
 488         return EFI_OUT_OF_RESOURCES;
 489       }
 490       Sq->Mptr = PhyAddr;
 491     }
 492   }
 493   //
 494   // If the buffer size spans more than two memory pages (page size as defined in CC.Mps),
 495   // then build a PRP list in the second PRP submission queue entry.
 496   //
 497   Offset = ((UINT16)Sq->Prp[0]) & (EFI_PAGE_SIZE - 1);
 498   Bytes  = Packet->TransferLength;
 499
 500   if ((Offset + Bytes) > (EFI_PAGE_SIZE * 2)) {
 501     //
 502     // Create PrpList for remaining data buffer.
 503     //
 504     PhyAddr = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
 505     Prp = NvmeCreatePrpList (PciIo, PhyAddr, EFI_SIZE_TO_PAGES(Offset + Bytes) - 1, &PrpListHost, &PrpListNo, &MapPrpList);
 506     if (Prp == NULL) {
 507       goto EXIT;
 508     }
 509
 510     Sq->Prp[1] = (UINT64)(UINTN)Prp;
 511   } else if ((Offset + Bytes) > EFI_PAGE_SIZE) {
 512     Sq->Prp[1] = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
 513   }
 514
 515   if(Packet->NvmeCmd->Flags & CDW10_VALID) {
 516     Sq->Payload.Raw.Cdw10 = Packet->NvmeCmd->Cdw10;
 517   }
 518   if(Packet->NvmeCmd->Flags & CDW11_VALID) {
 519     Sq->Payload.Raw.Cdw11 = Packet->NvmeCmd->Cdw11;
 520   }
 521   if(Packet->NvmeCmd->Flags & CDW12_VALID) {
 522     Sq->Payload.Raw.Cdw12 = Packet->NvmeCmd->Cdw12;
 523   }
 524   if(Packet->NvmeCmd->Flags & CDW13_VALID) {
 525     Sq->Payload.Raw.Cdw13 = Packet->NvmeCmd->Cdw13;
 526   }
 527   if(Packet->NvmeCmd->Flags & CDW14_VALID) {
 528     Sq->Payload.Raw.Cdw14 = Packet->NvmeCmd->Cdw14;
 529   }
 530   if(Packet->NvmeCmd->Flags & CDW15_VALID) {
 531     Sq->Payload.Raw.Cdw15 = Packet->NvmeCmd->Cdw15;
 532   }
 533
 534   //
 535   // Ring the submission queue doorbell.
 536   //
 537   Private->SqTdbl[Qid].Sqt ^= 1;
 538   Data = ReadUnaligned32 ((UINT32*)&Private->SqTdbl[Qid]);
 539   PciIo->Mem.Write (
 540                PciIo,
 541                EfiPciIoWidthUint32,
 542                NVME_BAR,
 543                NVME_SQTDBL_OFFSET(Qid, Private->Cap.Dstrd),
 544                1,
 545                &Data
 546                );
 547
 548   Status = gBS->CreateEvent (
 549                   EVT_TIMER,
 550                   TPL_CALLBACK,
 551                   NULL,
 552                   NULL,
 553                   &TimerEvent
 554                   );
 555   if (EFI_ERROR (Status)) {
 556     goto EXIT;
 557   }
 558
 559   Status = gBS->SetTimer(TimerEvent, TimerRelative, Packet->CommandTimeout);
 560
 561   if (EFI_ERROR(Status)) {
 562     Packet->ControllerStatus = NVM_EXPRESS_STATUS_CONTROLLER_DEVICE_ERROR;
 563     goto EXIT;
 564   }
 565
 566   //
 567   // Wait for completion queue to get filled in.
 568   //
 569   Status = EFI_TIMEOUT;
 570   Packet->ControllerStatus = NVM_EXPRESS_STATUS_CONTROLLER_TIMEOUT_COMMAND;
 571   while (EFI_ERROR (gBS->CheckEvent (TimerEvent))) {
 572     if (Cq->Pt != Private->Pt[Qid]) {
 573       Status = EFI_SUCCESS;
 574       Packet->ControllerStatus = NVM_EXPRESS_STATUS_CONTROLLER_READY;
 575       break;
 576     }
 577   }
 578
 579   if ((Private->CqHdbl[Qid].Cqh ^= 1) == 0) {
 580     Private->Pt[Qid] ^= 1;
 581   }
 582
 583   //
 584   // Copy the Respose Queue entry for this command to the callers response buffer
 585   //
 586   CopyMem(Packet->NvmeResponse, Cq, sizeof(NVM_EXPRESS_RESPONSE));
 587
 588   //
 589   // Dump every completion entry status for debugging.
 590   //
 591   DEBUG_CODE_BEGIN();
 592     NvmeDumpStatus(Cq);
 593   DEBUG_CODE_END();
 594
 595   Data = ReadUnaligned32 ((UINT32*)&Private->CqHdbl[Qid]);
 596   PciIo->Mem.Write (
 597                PciIo,
 598                EfiPciIoWidthUint32,
 599                NVME_BAR,
 600                NVME_CQHDBL_OFFSET(Qid, Private->Cap.Dstrd),
 601                1,
 602                &Data
 603                );
 604
 605 EXIT:
 606   if (MapData != NULL) {
 607     PciIo->Unmap (
 608              PciIo,
 609              MapData
 610              );
 611   }
 612
 613   if (MapMeta != NULL) {
 614     PciIo->Unmap (
 615              PciIo,
 616              MapMeta
 617              );
 618   }
 619
 620   if (MapPrpList != NULL) {
 621     PciIo->Unmap (
 622              PciIo,
 623              MapPrpList
 624              );
 625   }
 626
 627   if (Prp != NULL) {
 628     PciIo->FreeBuffer (PciIo, PrpListNo, PrpListHost);
 629   }
 630
 631   if (TimerEvent != NULL) {
 632     gBS->CloseEvent (TimerEvent);
 633   }
 634   return Status;
 635 }
 636
 637 /**
 638   Used to retrieve the list of namespaces defined on an NVM Express controller.
 639
 640   The NVM_EXPRESS_PASS_THRU_PROTOCOL.GetNextNamespace() function retrieves a list of namespaces
 641   defined on an NVM Express controller. If on input a NamespaceID is specified by all 0xFF in the
 642   namespace buffer, then the first namespace defined on the NVM Express controller is returned in
 643   NamespaceID, and a status of EFI_SUCCESS is returned.
 644
 645   If NamespaceId is a Namespace value that was returned on a previous call to GetNextNamespace(),
 646   then the next valid NamespaceId  for an NVM Express SSD namespace on the NVM Express controller
 647   is returned in NamespaceId, and EFI_SUCCESS is returned.
 648
 649   If Namespace array is not a 0xFFFFFFFF and NamespaceId was not returned on a previous call to
 650   GetNextNamespace(), then EFI_INVALID_PARAMETER is returned.
 651
 652   If NamespaceId is the NamespaceId of the last SSD namespace on the NVM Express controller, then
 653   EFI_NOT_FOUND is returned
 654
 655   @param[in]     This           A pointer to the NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
 656   @param[in,out] NamespaceId    On input, a pointer to a legal NamespaceId for an NVM Express
 657                                 namespace present on the NVM Express controller. On output, a
 658                                 pointer to the next NamespaceId of an NVM Express namespace on
 659                                 an NVM Express controller. An input value of 0xFFFFFFFF retrieves
 660                                 the first NamespaceId for an NVM Express namespace present on an
 661                                 NVM Express controller.
 662   @param[out]    NamespaceUuid  On output, the UUID associated with the next namespace, if a UUID
 663                                 is defined for that NamespaceId, otherwise, zero is returned in
 664                                 this parameter. If the caller does not require a UUID, then a NULL
 665                                 pointer may be passed.
 666
 667   @retval EFI_SUCCESS           The NamespaceId of the next Namespace was returned.
 668   @retval EFI_NOT_FOUND         There are no more namespaces defined on this controller.
 669   @retval EFI_INVALID_PARAMETER Namespace array is not a 0xFFFFFFFF and NamespaceId was not returned
 670                                 on a previous call to GetNextNamespace().
 671
 672 **/
 673 EFI_STATUS
 674 EFIAPI
 675 NvmExpressGetNextNamespace (
 676   IN     NVM_EXPRESS_PASS_THRU_PROTOCOL              *This,
 677   IN OUT UINT32                                      *NamespaceId,
 678      OUT UINT64                                      *NamespaceUuid  OPTIONAL
 679   )
 680 {
 681   NVME_CONTROLLER_PRIVATE_DATA     *Private;
 682   NVME_ADMIN_NAMESPACE_DATA        *NamespaceData;
 683   UINT32                           NextNamespaceId;
 684   EFI_STATUS                       Status;
 685
 686   if ((This == NULL) || (NamespaceId == NULL)) {
 687     return EFI_INVALID_PARAMETER;
 688   }
 689
 690   NamespaceData = NULL;
 691   Status        = EFI_NOT_FOUND;
 692
 693   Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
 694   //
 695   // If the NamespaceId input value is 0xFFFFFFFF, then get the first valid namespace ID
 696   //
 697   if (*NamespaceId == 0xFFFFFFFF) {
 698     //
 699     // Start with the first namespace ID
 700     //
 701     NextNamespaceId = 1;
 702     //
 703     // Allocate buffer for Identify Namespace data.
 704     //
 705     NamespaceData = (NVME_ADMIN_NAMESPACE_DATA *)AllocateZeroPool (sizeof (NVME_ADMIN_NAMESPACE_DATA));
 706
 707     if (NamespaceData == NULL) {
 708       return EFI_NOT_FOUND;
 709     }
 710
 711     Status = NvmeIdentifyNamespace (Private, NextNamespaceId, NamespaceData);
 712     if (EFI_ERROR(Status)) {
 713       goto Done;
 714     }
 715
 716     *NamespaceId = NextNamespaceId;
 717     if (NamespaceUuid != NULL) {
 718       *NamespaceUuid = NamespaceData->Eui64;
 719     }
 720   } else {
 721     if (*NamespaceId >= Private->ControllerData->Nn) {
 722       return EFI_INVALID_PARAMETER;
 723     }
 724
 725     NextNamespaceId = *NamespaceId + 1;
 726     //
 727     // Allocate buffer for Identify Namespace data.
 728     //
 729     NamespaceData = (NVME_ADMIN_NAMESPACE_DATA *)AllocateZeroPool (sizeof (NVME_ADMIN_NAMESPACE_DATA));
 730     if (NamespaceData == NULL) {
 731       return EFI_NOT_FOUND;
 732     }
 733
 734     Status = NvmeIdentifyNamespace (Private, NextNamespaceId, NamespaceData);
 735     if (EFI_ERROR(Status)) {
 736       goto Done;
 737     }
 738
 739     *NamespaceId = NextNamespaceId;
 740     if (NamespaceUuid != NULL) {
 741       *NamespaceUuid = NamespaceData->Eui64;
 742     }
 743   }
 744
 745 Done:
 746   if (NamespaceData != NULL) {
 747     FreePool(NamespaceData);
 748   }
 749
 750   return Status;
 751 }
 752
 753 /**
 754   Used to translate a device path node to a Namespace ID and Namespace UUID.
 755
 756   The NVM_EXPRESS_PASS_THRU_PROTOCOL.GetNamwspace() function determines the Namespace ID and Namespace UUID
 757   associated with the NVM Express SSD namespace described by DevicePath. If DevicePath is a device path node type
 758   that the NVM Express Pass Thru driver supports, then the NVM Express Pass Thru driver will attempt to translate
 759   the contents DevicePath into a Namespace ID and UUID. If this translation is successful, then that Namespace ID
 760   and UUID are returned in NamespaceID and NamespaceUUID, and EFI_SUCCESS is returned.
 761
 762   @param[in]  This                A pointer to the NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
 763   @param[in]  DevicePath          A pointer to the device path node that describes an NVM Express namespace on
 764                                   the NVM Express controller.
 765   @param[out] NamespaceId         The NVM Express namespace ID contained in the device path node.
 766   @param[out] NamespaceUuid       The NVM Express namespace contained in the device path node.
 767
 768   @retval EFI_SUCCESS             DevicePath was successfully translated to NamespaceId and NamespaceUuid.
 769   @retval EFI_INVALID_PARAMETER   If DevicePath, NamespaceId, or NamespaceUuid are NULL, then EFI_INVALID_PARAMETER
 770                                   is returned.
 771   @retval EFI_UNSUPPORTED         If DevicePath is not a device path node type that the NVM Express Pass Thru driver
 772                                   supports, then EFI_UNSUPPORTED is returned.
 773   @retval EFI_NOT_FOUND           If DevicePath is a device path node type that the Nvm Express Pass Thru driver
 774                                   supports, but there is not a valid translation from DevicePath to a NamespaceID
 775                                   and NamespaceUuid, then EFI_NOT_FOUND is returned.
 776 **/
 777 EFI_STATUS
 778 EFIAPI
 779 NvmExpressGetNamespace (
 780   IN     NVM_EXPRESS_PASS_THRU_PROTOCOL              *This,
 781   IN     EFI_DEVICE_PATH_PROTOCOL                    *DevicePath,
 782      OUT UINT32                                      *NamespaceId,
 783      OUT UINT64                                      *NamespaceUuid
 784   )
 785 {
 786   NVME_NAMESPACE_DEVICE_PATH       *Node;
 787
 788   if ((This == NULL) || (DevicePath == NULL) || (NamespaceId == NULL) || (NamespaceUuid == NULL)) {
 789     return EFI_INVALID_PARAMETER;
 790   }
 791
 792   if (DevicePath->Type != MESSAGING_DEVICE_PATH) {
 793     return EFI_UNSUPPORTED;
 794   }
 795
 796   Node = (NVME_NAMESPACE_DEVICE_PATH *)DevicePath;
 797
 798   if (DevicePath->SubType == MSG_NVME_NAMESPACE_DP) {
 799     if (DevicePathNodeLength(DevicePath) != sizeof(NVME_NAMESPACE_DEVICE_PATH)) {
 800       return EFI_NOT_FOUND;
 801     }
 802
 803     *NamespaceId   = Node->NamespaceId;
 804     *NamespaceUuid = Node->NamespaceUuid;
 805
 806     return EFI_SUCCESS;
 807   } else {
 808     return EFI_UNSUPPORTED;
 809   }
 810 }
 811
 812 /**
 813   Used to allocate and build a device path node for an NVM Express namespace on an NVM Express controller.
 814
 815   The NVM_EXPRESS_PASS_THRU_PROTOCOL.BuildDevicePath() function allocates and builds a single device
 816   path node for the NVM Express namespace specified by NamespaceId.
 817
 818   If the namespace device specified by NamespaceId is not valid , then EFI_NOT_FOUND is returned.
 819
 820   If DevicePath is NULL, then EFI_INVALID_PARAMETER is returned.
 821
 822   If there are not enough resources to allocate the device path node, then EFI_OUT_OF_RESOURCES is returned.
 823
 824   Otherwise, DevicePath is allocated with the boot service AllocatePool(), the contents of DevicePath are
 825   initialized to describe the NVM Express namespace specified by NamespaceId, and EFI_SUCCESS is returned.
 826
 827   @param[in]     This                A pointer to the NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
 828   @param[in]     NamespaceId         The NVM Express namespace ID  for which a device path node is to be
 829                                      allocated and built. Caller must set the NamespaceId to zero if the
 830                                      device path node will contain a valid UUID.
 831   @param[in]     NamespaceUuid       The NVM Express namespace UUID for which a device path node is to be
 832                                      allocated and built. UUID will only be valid of the Namespace ID is zero.
 833   @param[in,out] DevicePath          A pointer to a single device path node that describes the NVM Express
 834                                      namespace specified by NamespaceId. This function is responsible for
 835                                      allocating the buffer DevicePath with the boot service AllocatePool().
 836                                      It is the caller's responsibility to free DevicePath when the caller
 837                                      is finished with DevicePath.
 838   @retval EFI_SUCCESS                The device path node that describes the NVM Express namespace specified
 839                                      by NamespaceId was allocated and returned in DevicePath.
 840   @retval EFI_NOT_FOUND              The NVM Express namespace specified by NamespaceId does not exist on the
 841                                      NVM Express controller.
 842   @retval EFI_INVALID_PARAMETER      DevicePath is NULL.
 843   @retval EFI_OUT_OF_RESOURCES       There are not enough resources to allocate the DevicePath node.
 844
 845 **/
 846 EFI_STATUS
 847 EFIAPI
 848 NvmExpressBuildDevicePath (
 849   IN     NVM_EXPRESS_PASS_THRU_PROTOCOL              *This,
 850   IN     UINT32                                      NamespaceId,
 851   IN     UINT64                                      NamespaceUuid,
 852   IN OUT EFI_DEVICE_PATH_PROTOCOL                    **DevicePath
 853   )
 854 {
 855   NVME_CONTROLLER_PRIVATE_DATA   *Private;
 856   NVME_NAMESPACE_DEVICE_PATH     *Node;
 857
 858   //
 859   // Validate parameters
 860   //
 861   if ((This == NULL) || (DevicePath == NULL)) {
 862     return EFI_INVALID_PARAMETER;
 863   }
 864
 865   Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
 866
 867   if (NamespaceId == 0) {
 868     return EFI_NOT_FOUND;
 869   }
 870
 871   Node = (NVME_NAMESPACE_DEVICE_PATH *)AllocateZeroPool (sizeof (NVME_NAMESPACE_DEVICE_PATH));
 872
 873   if (Node == NULL) {
 874     return EFI_OUT_OF_RESOURCES;
 875   }
 876
 877   Node->Header.Type    = MESSAGING_DEVICE_PATH;
 878   Node->Header.SubType = MSG_NVME_NAMESPACE_DP;
 879   SetDevicePathNodeLength (&Node->Header, sizeof (NVME_NAMESPACE_DEVICE_PATH));
 880   Node->NamespaceId    = NamespaceId;
 881   Node->NamespaceUuid  = NamespaceUuid;
 882
 883   *DevicePath = (EFI_DEVICE_PATH_PROTOCOL *)Node;
 884   return EFI_SUCCESS;
 885 }
 886