MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressPassthru.c

   1 /** @file
   2   NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows
   3   NVM Express specification.
   4
   5   (C) Copyright 2014 Hewlett-Packard Development Company, L.P.<BR>
   6   Copyright (c) 2013 - 2016, Intel Corporation. All rights reserved.<BR>
   7   This program and the accompanying materials
   8   are licensed and made available under the terms and conditions of the BSD License
   9   which accompanies this distribution.  The full text of the license may be found at
  10   http://opensource.org/licenses/bsd-license.php.
  11
  12   THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  13   WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  14
  15 **/
  16
  17 #include "NvmExpress.h"
  18
  19 /**
  20   Dump the execution status from a given completion queue entry.
  21
  22   @param[in]     Cq               A pointer to the NVME_CQ item.
  23
  24 **/
  25 VOID
  26 NvmeDumpStatus (
  27   IN NVME_CQ             *Cq
  28   )
  29 {
  30   DEBUG ((EFI_D_VERBOSE, "Dump NVMe Completion Entry Status from [0x%x]:\n", Cq));
  31
  32   DEBUG ((EFI_D_VERBOSE, "  SQ Identifier : [0x%x], Phase Tag : [%d], Cmd Identifier : [0x%x]\n", Cq->Sqid, Cq->Pt, Cq->Cid));
  33
  34   DEBUG ((EFI_D_VERBOSE, "  NVMe Cmd Execution Result - "));
  35
  36   switch (Cq->Sct) {
  37     case 0x0:
  38       switch (Cq->Sc) {
  39         case 0x0:
  40           DEBUG ((EFI_D_VERBOSE, "Successful Completion\n"));
  41           break;
  42         case 0x1:
  43           DEBUG ((EFI_D_VERBOSE, "Invalid Command Opcode\n"));
  44           break;
  45         case 0x2:
  46           DEBUG ((EFI_D_VERBOSE, "Invalid Field in Command\n"));
  47           break;
  48         case 0x3:
  49           DEBUG ((EFI_D_VERBOSE, "Command ID Conflict\n"));
  50           break;
  51         case 0x4:
  52           DEBUG ((EFI_D_VERBOSE, "Data Transfer Error\n"));
  53           break;
  54         case 0x5:
  55           DEBUG ((EFI_D_VERBOSE, "Commands Aborted due to Power Loss Notification\n"));
  56           break;
  57         case 0x6:
  58           DEBUG ((EFI_D_VERBOSE, "Internal Device Error\n"));
  59           break;
  60         case 0x7:
  61           DEBUG ((EFI_D_VERBOSE, "Command Abort Requested\n"));
  62           break;
  63         case 0x8:
  64           DEBUG ((EFI_D_VERBOSE, "Command Aborted due to SQ Deletion\n"));
  65           break;
  66         case 0x9:
  67           DEBUG ((EFI_D_VERBOSE, "Command Aborted due to Failed Fused Command\n"));
  68           break;
  69         case 0xA:
  70           DEBUG ((EFI_D_VERBOSE, "Command Aborted due to Missing Fused Command\n"));
  71           break;
  72         case 0xB:
  73           DEBUG ((EFI_D_VERBOSE, "Invalid Namespace or Format\n"));
  74           break;
  75         case 0xC:
  76           DEBUG ((EFI_D_VERBOSE, "Command Sequence Error\n"));
  77           break;
  78         case 0xD:
  79           DEBUG ((EFI_D_VERBOSE, "Invalid SGL Last Segment Descriptor\n"));
  80           break;
  81         case 0xE:
  82           DEBUG ((EFI_D_VERBOSE, "Invalid Number of SGL Descriptors\n"));
  83           break;
  84         case 0xF:
  85           DEBUG ((EFI_D_VERBOSE, "Data SGL Length Invalid\n"));
  86           break;
  87         case 0x10:
  88           DEBUG ((EFI_D_VERBOSE, "Metadata SGL Length Invalid\n"));
  89           break;
  90         case 0x11:
  91           DEBUG ((EFI_D_VERBOSE, "SGL Descriptor Type Invalid\n"));
  92           break;
  93         case 0x80:
  94           DEBUG ((EFI_D_VERBOSE, "LBA Out of Range\n"));
  95           break;
  96         case 0x81:
  97           DEBUG ((EFI_D_VERBOSE, "Capacity Exceeded\n"));
  98           break;
  99         case 0x82:
 100           DEBUG ((EFI_D_VERBOSE, "Namespace Not Ready\n"));
 101           break;
 102         case 0x83:
 103           DEBUG ((EFI_D_VERBOSE, "Reservation Conflict\n"));
 104           break;
 105       }
 106       break;
 107
 108     case 0x1:
 109       switch (Cq->Sc) {
 110         case 0x0:
 111           DEBUG ((EFI_D_VERBOSE, "Completion Queue Invalid\n"));
 112           break;
 113         case 0x1:
 114           DEBUG ((EFI_D_VERBOSE, "Invalid Queue Identifier\n"));
 115           break;
 116         case 0x2:
 117           DEBUG ((EFI_D_VERBOSE, "Maximum Queue Size Exceeded\n"));
 118           break;
 119         case 0x3:
 120           DEBUG ((EFI_D_VERBOSE, "Abort Command Limit Exceeded\n"));
 121           break;
 122         case 0x5:
 123           DEBUG ((EFI_D_VERBOSE, "Asynchronous Event Request Limit Exceeded\n"));
 124           break;
 125         case 0x6:
 126           DEBUG ((EFI_D_VERBOSE, "Invalid Firmware Slot\n"));
 127           break;
 128         case 0x7:
 129           DEBUG ((EFI_D_VERBOSE, "Invalid Firmware Image\n"));
 130           break;
 131         case 0x8:
 132           DEBUG ((EFI_D_VERBOSE, "Invalid Interrupt Vector\n"));
 133           break;
 134         case 0x9:
 135           DEBUG ((EFI_D_VERBOSE, "Invalid Log Page\n"));
 136           break;
 137         case 0xA:
 138           DEBUG ((EFI_D_VERBOSE, "Invalid Format\n"));
 139           break;
 140         case 0xB:
 141           DEBUG ((EFI_D_VERBOSE, "Firmware Application Requires Conventional Reset\n"));
 142           break;
 143         case 0xC:
 144           DEBUG ((EFI_D_VERBOSE, "Invalid Queue Deletion\n"));
 145           break;
 146         case 0xD:
 147           DEBUG ((EFI_D_VERBOSE, "Feature Identifier Not Saveable\n"));
 148           break;
 149         case 0xE:
 150           DEBUG ((EFI_D_VERBOSE, "Feature Not Changeable\n"));
 151           break;
 152         case 0xF:
 153           DEBUG ((EFI_D_VERBOSE, "Feature Not Namespace Specific\n"));
 154           break;
 155         case 0x10:
 156           DEBUG ((EFI_D_VERBOSE, "Firmware Application Requires NVM Subsystem Reset\n"));
 157           break;
 158         case 0x80:
 159           DEBUG ((EFI_D_VERBOSE, "Conflicting Attributes\n"));
 160           break;
 161         case 0x81:
 162           DEBUG ((EFI_D_VERBOSE, "Invalid Protection Information\n"));
 163           break;
 164         case 0x82:
 165           DEBUG ((EFI_D_VERBOSE, "Attempted Write to Read Only Range\n"));
 166           break;
 167       }
 168       break;
 169
 170     case 0x2:
 171       switch (Cq->Sc) {
 172         case 0x80:
 173           DEBUG ((EFI_D_VERBOSE, "Write Fault\n"));
 174           break;
 175         case 0x81:
 176           DEBUG ((EFI_D_VERBOSE, "Unrecovered Read Error\n"));
 177           break;
 178         case 0x82:
 179           DEBUG ((EFI_D_VERBOSE, "End-to-end Guard Check Error\n"));
 180           break;
 181         case 0x83:
 182           DEBUG ((EFI_D_VERBOSE, "End-to-end Application Tag Check Error\n"));
 183           break;
 184         case 0x84:
 185           DEBUG ((EFI_D_VERBOSE, "End-to-end Reference Tag Check Error\n"));
 186           break;
 187         case 0x85:
 188           DEBUG ((EFI_D_VERBOSE, "Compare Failure\n"));
 189           break;
 190         case 0x86:
 191           DEBUG ((EFI_D_VERBOSE, "Access Denied\n"));
 192           break;
 193       }
 194       break;
 195
 196     default:
 197       break;
 198   }
 199 }
 200
 201 /**
 202   Create PRP lists for data transfer which is larger than 2 memory pages.
 203   Note here we calcuate the number of required PRP lists and allocate them at one time.
 204
 205   @param[in]     PciIo               A pointer to the EFI_PCI_IO_PROTOCOL instance.
 206   @param[in]     PhysicalAddr        The physical base address of data buffer.
 207   @param[in]     Pages               The number of pages to be transfered.
 208   @param[out]    PrpListHost         The host base address of PRP lists.
 209   @param[in,out] PrpListNo           The number of PRP List.
 210   @param[out]    Mapping             The mapping value returned from PciIo.Map().
 211
 212   @retval The pointer to the first PRP List of the PRP lists.
 213
 214 **/
 215 VOID*
 216 NvmeCreatePrpList (
 217   IN     EFI_PCI_IO_PROTOCOL          *PciIo,
 218   IN     EFI_PHYSICAL_ADDRESS         PhysicalAddr,
 219   IN     UINTN                        Pages,
 220      OUT VOID                         **PrpListHost,
 221   IN OUT UINTN                        *PrpListNo,
 222      OUT VOID                         **Mapping
 223   )
 224 {
 225   UINTN                       PrpEntryNo;
 226   UINT64                      PrpListBase;
 227   UINTN                       PrpListIndex;
 228   UINTN                       PrpEntryIndex;
 229   UINT64                      Remainder;
 230   EFI_PHYSICAL_ADDRESS        PrpListPhyAddr;
 231   UINTN                       Bytes;
 232   EFI_STATUS                  Status;
 233
 234   //
 235   // The number of Prp Entry in a memory page.
 236   //
 237   PrpEntryNo = EFI_PAGE_SIZE / sizeof (UINT64);
 238
 239   //
 240   // Calculate total PrpList number.
 241   //
 242   *PrpListNo = (UINTN)DivU64x64Remainder ((UINT64)Pages, (UINT64)PrpEntryNo - 1, &Remainder);
 243   if (*PrpListNo == 0) {
 244     *PrpListNo = 1;
 245   } else if ((Remainder != 0) && (Remainder != 1)) {
 246     *PrpListNo += 1;
 247   } else if (Remainder == 1) {
 248     Remainder = PrpEntryNo;
 249   } else if (Remainder == 0) {
 250     Remainder = PrpEntryNo - 1;
 251   }
 252
 253   Status = PciIo->AllocateBuffer (
 254                     PciIo,
 255                     AllocateAnyPages,
 256                     EfiBootServicesData,
 257                     *PrpListNo,
 258                     PrpListHost,
 259                     0
 260                     );
 261
 262   if (EFI_ERROR (Status)) {
 263     return NULL;
 264   }
 265
 266   Bytes = EFI_PAGES_TO_SIZE (*PrpListNo);
 267   Status = PciIo->Map (
 268                     PciIo,
 269                     EfiPciIoOperationBusMasterCommonBuffer,
 270                     *PrpListHost,
 271                     &Bytes,
 272                     &PrpListPhyAddr,
 273                     Mapping
 274                     );
 275
 276   if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (*PrpListNo))) {
 277     DEBUG ((EFI_D_ERROR, "NvmeCreatePrpList: create PrpList failure!\n"));
 278     goto EXIT;
 279   }
 280   //
 281   // Fill all PRP lists except of last one.
 282   //
 283   ZeroMem (*PrpListHost, Bytes);
 284   for (PrpListIndex = 0; PrpListIndex < *PrpListNo - 1; ++PrpListIndex) {
 285     PrpListBase = *(UINT64*)PrpListHost + PrpListIndex * EFI_PAGE_SIZE;
 286
 287     for (PrpEntryIndex = 0; PrpEntryIndex < PrpEntryNo; ++PrpEntryIndex) {
 288       if (PrpEntryIndex != PrpEntryNo - 1) {
 289         //
 290         // Fill all PRP entries except of last one.
 291         //
 292         *((UINT64*)(UINTN)PrpListBase + PrpEntryIndex) = PhysicalAddr;
 293         PhysicalAddr += EFI_PAGE_SIZE;
 294       } else {
 295         //
 296         // Fill last PRP entries with next PRP List pointer.
 297         //
 298         *((UINT64*)(UINTN)PrpListBase + PrpEntryIndex) = PrpListPhyAddr + (PrpListIndex + 1) * EFI_PAGE_SIZE;
 299       }
 300     }
 301   }
 302   //
 303   // Fill last PRP list.
 304   //
 305   PrpListBase = *(UINT64*)PrpListHost + PrpListIndex * EFI_PAGE_SIZE;
 306   for (PrpEntryIndex = 0; PrpEntryIndex < Remainder; ++PrpEntryIndex) {
 307     *((UINT64*)(UINTN)PrpListBase + PrpEntryIndex) = PhysicalAddr;
 308     PhysicalAddr += EFI_PAGE_SIZE;
 309   }
 310
 311   return (VOID*)(UINTN)PrpListPhyAddr;
 312
 313 EXIT:
 314   PciIo->FreeBuffer (PciIo, *PrpListNo, *PrpListHost);
 315   return NULL;
 316 }
 317
 318
 319 /**
 320   Sends an NVM Express Command Packet to an NVM Express controller or namespace. This function supports
 321   both blocking I/O and non-blocking I/O. The blocking I/O functionality is required, and the non-blocking
 322   I/O functionality is optional.
 323
 324
 325   @param[in]     This                A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
 326   @param[in]     NamespaceId         A 32 bit namespace ID as defined in the NVMe specification to which the NVM Express Command
 327                                      Packet will be sent.  A value of 0 denotes the NVM Express controller, a value of all 0xFF's
 328                                      (all bytes are 0xFF) in the namespace ID specifies that the command packet should be sent to
 329                                      all valid namespaces.
 330   @param[in,out] Packet              A pointer to the NVM Express Command Packet.
 331   @param[in]     Event               If non-blocking I/O is not supported then Event is ignored, and blocking I/O is performed.
 332                                      If Event is NULL, then blocking I/O is performed. If Event is not NULL and non-blocking I/O
 333                                      is supported, then non-blocking I/O is performed, and Event will be signaled when the NVM
 334                                      Express Command Packet completes.
 335
 336   @retval EFI_SUCCESS                The NVM Express Command Packet was sent by the host. TransferLength bytes were transferred
 337                                      to, or from DataBuffer.
 338   @retval EFI_BAD_BUFFER_SIZE        The NVM Express Command Packet was not executed. The number of bytes that could be transferred
 339                                      is returned in TransferLength.
 340   @retval EFI_NOT_READY              The NVM Express Command Packet could not be sent because the controller is not ready. The caller
 341                                      may retry again later.
 342   @retval EFI_DEVICE_ERROR           A device error occurred while attempting to send the NVM Express Command Packet.
 343   @retval EFI_INVALID_PARAMETER      NamespaceId or the contents of EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET are invalid. The NVM
 344                                      Express Command Packet was not sent, so no additional status information is available.
 345   @retval EFI_UNSUPPORTED            The command described by the NVM Express Command Packet is not supported by the NVM Express
 346                                      controller. The NVM Express Command Packet was not sent so no additional status information
 347                                      is available.
 348   @retval EFI_TIMEOUT                A timeout occurred while waiting for the NVM Express Command Packet to execute.
 349
 350 **/
 351 EFI_STATUS
 352 EFIAPI
 353 NvmExpressPassThru (
 354   IN     EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL          *This,
 355   IN     UINT32                                      NamespaceId,
 356   IN OUT EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET    *Packet,
 357   IN     EFI_EVENT                                   Event OPTIONAL
 358   )
 359 {
 360   NVME_CONTROLLER_PRIVATE_DATA  *Private;
 361   EFI_STATUS                    Status;
 362   EFI_PCI_IO_PROTOCOL           *PciIo;
 363   NVME_SQ                       *Sq;
 364   NVME_CQ                       *Cq;
 365   UINT16                        QueueId;
 366   UINT32                        Bytes;
 367   UINT16                        Offset;
 368   EFI_EVENT                     TimerEvent;
 369   EFI_PCI_IO_PROTOCOL_OPERATION Flag;
 370   EFI_PHYSICAL_ADDRESS          PhyAddr;
 371   VOID                          *MapData;
 372   VOID                          *MapMeta;
 373   VOID                          *MapPrpList;
 374   UINTN                         MapLength;
 375   UINT64                        *Prp;
 376   VOID                          *PrpListHost;
 377   UINTN                         PrpListNo;
 378   UINT32                        Data;
 379   NVME_PASS_THRU_ASYNC_REQ      *AsyncRequest;
 380   EFI_TPL                       OldTpl;
 381
 382   //
 383   // check the data fields in Packet parameter.
 384   //
 385   if ((This == NULL) || (Packet == NULL)) {
 386     return EFI_INVALID_PARAMETER;
 387   }
 388
 389   if ((Packet->NvmeCmd == NULL) || (Packet->NvmeCompletion == NULL)) {
 390     return EFI_INVALID_PARAMETER;
 391   }
 392
 393   if (Packet->QueueType != NVME_ADMIN_QUEUE && Packet->QueueType != NVME_IO_QUEUE) {
 394     return EFI_INVALID_PARAMETER;
 395   }
 396
 397   Private     = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
 398   PciIo       = Private->PciIo;
 399   MapData     = NULL;
 400   MapMeta     = NULL;
 401   MapPrpList  = NULL;
 402   PrpListHost = NULL;
 403   PrpListNo   = 0;
 404   Prp         = NULL;
 405   TimerEvent  = NULL;
 406   Status      = EFI_SUCCESS;
 407
 408   if (Packet->QueueType == NVME_ADMIN_QUEUE) {
 409     QueueId = 0;
 410   } else {
 411     if (Event == NULL) {
 412       QueueId = 1;
 413     } else {
 414       QueueId = 2;
 415
 416       //
 417       // Submission queue full check.
 418       //
 419       if ((Private->SqTdbl[QueueId].Sqt + 1) % (NVME_ASYNC_CSQ_SIZE + 1) ==
 420           Private->AsyncSqHead) {
 421         return EFI_NOT_READY;
 422       }
 423     }
 424   }
 425   Sq  = Private->SqBuffer[QueueId] + Private->SqTdbl[QueueId].Sqt;
 426   Cq  = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
 427
 428   if (Packet->NvmeCmd->Nsid != NamespaceId) {
 429     return EFI_INVALID_PARAMETER;
 430   }
 431
 432   ZeroMem (Sq, sizeof (NVME_SQ));
 433   Sq->Opc  = (UINT8)Packet->NvmeCmd->Cdw0.Opcode;
 434   Sq->Fuse = (UINT8)Packet->NvmeCmd->Cdw0.FusedOperation;
 435   Sq->Cid  = Private->Cid[QueueId]++;
 436   Sq->Nsid = Packet->NvmeCmd->Nsid;
 437
 438   //
 439   // Currently we only support PRP for data transfer, SGL is NOT supported.
 440   //
 441   ASSERT (Sq->Psdt == 0);
 442   if (Sq->Psdt != 0) {
 443     DEBUG ((EFI_D_ERROR, "NvmExpressPassThru: doesn't support SGL mechanism\n"));
 444     return EFI_UNSUPPORTED;
 445   }
 446
 447   Sq->Prp[0] = (UINT64)(UINTN)Packet->TransferBuffer;
 448   //
 449   // If the NVMe cmd has data in or out, then mapping the user buffer to the PCI controller specific addresses.
 450   // Note here we don't handle data buffer for CreateIOSubmitionQueue and CreateIOCompletionQueue cmds because
 451   // these two cmds are special which requires their data buffer must support simultaneous access by both the
 452   // processor and a PCI Bus Master. It's caller's responsbility to ensure this.
 453   //
 454   if (((Sq->Opc & (BIT0 | BIT1)) != 0) && (Sq->Opc != NVME_ADMIN_CRIOCQ_CMD) && (Sq->Opc != NVME_ADMIN_CRIOSQ_CMD)) {
 455     if ((Sq->Opc & BIT0) != 0) {
 456       Flag = EfiPciIoOperationBusMasterRead;
 457     } else {
 458       Flag = EfiPciIoOperationBusMasterWrite;
 459     }
 460
 461     MapLength = Packet->TransferLength;
 462     Status = PciIo->Map (
 463                       PciIo,
 464                       Flag,
 465                       Packet->TransferBuffer,
 466                       &MapLength,
 467                       &PhyAddr,
 468                       &MapData
 469                       );
 470     if (EFI_ERROR (Status) || (Packet->TransferLength != MapLength)) {
 471       return EFI_OUT_OF_RESOURCES;
 472     }
 473
 474     Sq->Prp[0] = PhyAddr;
 475     Sq->Prp[1] = 0;
 476
 477     MapLength = Packet->MetadataLength;
 478     if(Packet->MetadataBuffer != NULL) {
 479       MapLength = Packet->MetadataLength;
 480       Status = PciIo->Map (
 481                         PciIo,
 482                         Flag,
 483                         Packet->MetadataBuffer,
 484                         &MapLength,
 485                         &PhyAddr,
 486                         &MapMeta
 487                         );
 488       if (EFI_ERROR (Status) || (Packet->MetadataLength != MapLength)) {
 489         PciIo->Unmap (
 490                  PciIo,
 491                  MapData
 492                  );
 493
 494         return EFI_OUT_OF_RESOURCES;
 495       }
 496       Sq->Mptr = PhyAddr;
 497     }
 498   }
 499   //
 500   // If the buffer size spans more than two memory pages (page size as defined in CC.Mps),
 501   // then build a PRP list in the second PRP submission queue entry.
 502   //
 503   Offset = ((UINT16)Sq->Prp[0]) & (EFI_PAGE_SIZE - 1);
 504   Bytes  = Packet->TransferLength;
 505
 506   if ((Offset + Bytes) > (EFI_PAGE_SIZE * 2)) {
 507     //
 508     // Create PrpList for remaining data buffer.
 509     //
 510     PhyAddr = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
 511     Prp = NvmeCreatePrpList (PciIo, PhyAddr, EFI_SIZE_TO_PAGES(Offset + Bytes) - 1, &PrpListHost, &PrpListNo, &MapPrpList);
 512     if (Prp == NULL) {
 513       goto EXIT;
 514     }
 515
 516     Sq->Prp[1] = (UINT64)(UINTN)Prp;
 517   } else if ((Offset + Bytes) > EFI_PAGE_SIZE) {
 518     Sq->Prp[1] = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
 519   }
 520
 521   if(Packet->NvmeCmd->Flags & CDW2_VALID) {
 522     Sq->Rsvd2 = (UINT64)Packet->NvmeCmd->Cdw2;
 523   }
 524   if(Packet->NvmeCmd->Flags & CDW3_VALID) {
 525     Sq->Rsvd2 |= LShiftU64 ((UINT64)Packet->NvmeCmd->Cdw3, 32);
 526   }
 527   if(Packet->NvmeCmd->Flags & CDW10_VALID) {
 528     Sq->Payload.Raw.Cdw10 = Packet->NvmeCmd->Cdw10;
 529   }
 530   if(Packet->NvmeCmd->Flags & CDW11_VALID) {
 531     Sq->Payload.Raw.Cdw11 = Packet->NvmeCmd->Cdw11;
 532   }
 533   if(Packet->NvmeCmd->Flags & CDW12_VALID) {
 534     Sq->Payload.Raw.Cdw12 = Packet->NvmeCmd->Cdw12;
 535   }
 536   if(Packet->NvmeCmd->Flags & CDW13_VALID) {
 537     Sq->Payload.Raw.Cdw13 = Packet->NvmeCmd->Cdw13;
 538   }
 539   if(Packet->NvmeCmd->Flags & CDW14_VALID) {
 540     Sq->Payload.Raw.Cdw14 = Packet->NvmeCmd->Cdw14;
 541   }
 542   if(Packet->NvmeCmd->Flags & CDW15_VALID) {
 543     Sq->Payload.Raw.Cdw15 = Packet->NvmeCmd->Cdw15;
 544   }
 545
 546   //
 547   // Ring the submission queue doorbell.
 548   //
 549   if (Event != NULL) {
 550     Private->SqTdbl[QueueId].Sqt =
 551       (Private->SqTdbl[QueueId].Sqt + 1) % (NVME_ASYNC_CSQ_SIZE + 1);
 552   } else {
 553     Private->SqTdbl[QueueId].Sqt ^= 1;
 554   }
 555   Data = ReadUnaligned32 ((UINT32*)&Private->SqTdbl[QueueId]);
 556   PciIo->Mem.Write (
 557                PciIo,
 558                EfiPciIoWidthUint32,
 559                NVME_BAR,
 560                NVME_SQTDBL_OFFSET(QueueId, Private->Cap.Dstrd),
 561                1,
 562                &Data
 563                );
 564
 565   //
 566   // For non-blocking requests, return directly if the command is placed
 567   // in the submission queue.
 568   //
 569   if (Event != NULL) {
 570     AsyncRequest = AllocateZeroPool (sizeof (NVME_PASS_THRU_ASYNC_REQ));
 571     if (AsyncRequest == NULL) {
 572       Status = EFI_DEVICE_ERROR;
 573       goto EXIT;
 574     }
 575
 576     AsyncRequest->Signature     = NVME_PASS_THRU_ASYNC_REQ_SIG;
 577     AsyncRequest->Packet        = Packet;
 578     AsyncRequest->CommandId     = Sq->Cid;
 579     AsyncRequest->CallerEvent   = Event;
 580
 581     OldTpl = gBS->RaiseTPL (TPL_NOTIFY);
 582     InsertTailList (&Private->AsyncPassThruQueue, &AsyncRequest->Link);
 583     gBS->RestoreTPL (OldTpl);
 584
 585     return EFI_SUCCESS;
 586   }
 587
 588   Status = gBS->CreateEvent (
 589                   EVT_TIMER,
 590                   TPL_CALLBACK,
 591                   NULL,
 592                   NULL,
 593                   &TimerEvent
 594                   );
 595   if (EFI_ERROR (Status)) {
 596     goto EXIT;
 597   }
 598
 599   Status = gBS->SetTimer(TimerEvent, TimerRelative, Packet->CommandTimeout);
 600
 601   if (EFI_ERROR(Status)) {
 602     goto EXIT;
 603   }
 604
 605   //
 606   // Wait for completion queue to get filled in.
 607   //
 608   Status = EFI_TIMEOUT;
 609   while (EFI_ERROR (gBS->CheckEvent (TimerEvent))) {
 610     if (Cq->Pt != Private->Pt[QueueId]) {
 611       Status = EFI_SUCCESS;
 612       break;
 613     }
 614   }
 615
 616   //
 617   // Check the NVMe cmd execution result
 618   //
 619   if (Status != EFI_TIMEOUT) {
 620     if ((Cq->Sct == 0) && (Cq->Sc == 0)) {
 621       Status = EFI_SUCCESS;
 622     } else {
 623       Status = EFI_DEVICE_ERROR;
 624       //
 625       // Copy the Respose Queue entry for this command to the callers response buffer
 626       //
 627       CopyMem(Packet->NvmeCompletion, Cq, sizeof(EFI_NVM_EXPRESS_COMPLETION));
 628
 629       //
 630       // Dump every completion entry status for debugging.
 631       //
 632       DEBUG_CODE_BEGIN();
 633         NvmeDumpStatus(Cq);
 634       DEBUG_CODE_END();
 635     }
 636   }
 637
 638   if ((Private->CqHdbl[QueueId].Cqh ^= 1) == 0) {
 639     Private->Pt[QueueId] ^= 1;
 640   }
 641
 642   Data = ReadUnaligned32 ((UINT32*)&Private->CqHdbl[QueueId]);
 643   PciIo->Mem.Write (
 644                PciIo,
 645                EfiPciIoWidthUint32,
 646                NVME_BAR,
 647                NVME_CQHDBL_OFFSET(QueueId, Private->Cap.Dstrd),
 648                1,
 649                &Data
 650                );
 651
 652 EXIT:
 653   if (MapData != NULL) {
 654     PciIo->Unmap (
 655              PciIo,
 656              MapData
 657              );
 658   }
 659
 660   if (MapMeta != NULL) {
 661     PciIo->Unmap (
 662              PciIo,
 663              MapMeta
 664              );
 665   }
 666
 667   if (MapPrpList != NULL) {
 668     PciIo->Unmap (
 669              PciIo,
 670              MapPrpList
 671              );
 672   }
 673
 674   if (Prp != NULL) {
 675     PciIo->FreeBuffer (PciIo, PrpListNo, PrpListHost);
 676   }
 677
 678   if (TimerEvent != NULL) {
 679     gBS->CloseEvent (TimerEvent);
 680   }
 681   return Status;
 682 }
 683
 684 /**
 685   Used to retrieve the next namespace ID for this NVM Express controller.
 686
 687   The EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL.GetNextNamespace() function retrieves the next valid
 688   namespace ID on this NVM Express controller.
 689
 690   If on input the value pointed to by NamespaceId is 0xFFFFFFFF, then the first valid namespace
 691   ID defined on the NVM Express controller is returned in the location pointed to by NamespaceId
 692   and a status of EFI_SUCCESS is returned.
 693
 694   If on input the value pointed to by NamespaceId is an invalid namespace ID other than 0xFFFFFFFF,
 695   then EFI_INVALID_PARAMETER is returned.
 696
 697   If on input the value pointed to by NamespaceId is a valid namespace ID, then the next valid
 698   namespace ID on the NVM Express controller is returned in the location pointed to by NamespaceId,
 699   and EFI_SUCCESS is returned.
 700
 701   If the value pointed to by NamespaceId is the namespace ID of the last namespace on the NVM
 702   Express controller, then EFI_NOT_FOUND is returned.
 703
 704   @param[in]     This           A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
 705   @param[in,out] NamespaceId    On input, a pointer to a legal NamespaceId for an NVM Express
 706                                 namespace present on the NVM Express controller. On output, a
 707                                 pointer to the next NamespaceId of an NVM Express namespace on
 708                                 an NVM Express controller. An input value of 0xFFFFFFFF retrieves
 709                                 the first NamespaceId for an NVM Express namespace present on an
 710                                 NVM Express controller.
 711
 712   @retval EFI_SUCCESS           The Namespace ID of the next Namespace was returned.
 713   @retval EFI_NOT_FOUND         There are no more namespaces defined on this controller.
 714   @retval EFI_INVALID_PARAMETER NamespaceId is an invalid value other than 0xFFFFFFFF.
 715
 716 **/
 717 EFI_STATUS
 718 EFIAPI
 719 NvmExpressGetNextNamespace (
 720   IN     EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL          *This,
 721   IN OUT UINT32                                      *NamespaceId
 722   )
 723 {
 724   NVME_CONTROLLER_PRIVATE_DATA     *Private;
 725   NVME_ADMIN_NAMESPACE_DATA        *NamespaceData;
 726   UINT32                           NextNamespaceId;
 727   EFI_STATUS                       Status;
 728
 729   if ((This == NULL) || (NamespaceId == NULL)) {
 730     return EFI_INVALID_PARAMETER;
 731   }
 732
 733   NamespaceData = NULL;
 734   Status        = EFI_NOT_FOUND;
 735
 736   Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
 737   //
 738   // If the NamespaceId input value is 0xFFFFFFFF, then get the first valid namespace ID
 739   //
 740   if (*NamespaceId == 0xFFFFFFFF) {
 741     //
 742     // Start with the first namespace ID
 743     //
 744     NextNamespaceId = 1;
 745     //
 746     // Allocate buffer for Identify Namespace data.
 747     //
 748     NamespaceData = (NVME_ADMIN_NAMESPACE_DATA *)AllocateZeroPool (sizeof (NVME_ADMIN_NAMESPACE_DATA));
 749
 750     if (NamespaceData == NULL) {
 751       return EFI_NOT_FOUND;
 752     }
 753
 754     Status = NvmeIdentifyNamespace (Private, NextNamespaceId, NamespaceData);
 755     if (EFI_ERROR(Status)) {
 756       goto Done;
 757     }
 758
 759     *NamespaceId = NextNamespaceId;
 760   } else {
 761     if (*NamespaceId >= Private->ControllerData->Nn) {
 762       return EFI_INVALID_PARAMETER;
 763     }
 764
 765     NextNamespaceId = *NamespaceId + 1;
 766     //
 767     // Allocate buffer for Identify Namespace data.
 768     //
 769     NamespaceData = (NVME_ADMIN_NAMESPACE_DATA *)AllocateZeroPool (sizeof (NVME_ADMIN_NAMESPACE_DATA));
 770     if (NamespaceData == NULL) {
 771       return EFI_NOT_FOUND;
 772     }
 773
 774     Status = NvmeIdentifyNamespace (Private, NextNamespaceId, NamespaceData);
 775     if (EFI_ERROR(Status)) {
 776       goto Done;
 777     }
 778
 779     *NamespaceId = NextNamespaceId;
 780   }
 781
 782 Done:
 783   if (NamespaceData != NULL) {
 784     FreePool(NamespaceData);
 785   }
 786
 787   return Status;
 788 }
 789
 790 /**
 791   Used to translate a device path node to a namespace ID.
 792
 793   The EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL.GetNamespace() function determines the namespace ID associated with the
 794   namespace described by DevicePath.
 795
 796   If DevicePath is a device path node type that the NVM Express Pass Thru driver supports, then the NVM Express
 797   Pass Thru driver will attempt to translate the contents DevicePath into a namespace ID.
 798
 799   If this translation is successful, then that namespace ID is returned in NamespaceId, and EFI_SUCCESS is returned
 800
 801   @param[in]  This                A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
 802   @param[in]  DevicePath          A pointer to the device path node that describes an NVM Express namespace on
 803                                   the NVM Express controller.
 804   @param[out] NamespaceId         The NVM Express namespace ID contained in the device path node.
 805
 806   @retval EFI_SUCCESS             DevicePath was successfully translated to NamespaceId.
 807   @retval EFI_INVALID_PARAMETER   If DevicePath or NamespaceId are NULL, then EFI_INVALID_PARAMETER is returned.
 808   @retval EFI_UNSUPPORTED         If DevicePath is not a device path node type that the NVM Express Pass Thru driver
 809                                   supports, then EFI_UNSUPPORTED is returned.
 810   @retval EFI_NOT_FOUND           If DevicePath is a device path node type that the NVM Express Pass Thru driver
 811                                   supports, but there is not a valid translation from DevicePath to a namespace ID,
 812                                   then EFI_NOT_FOUND is returned.
 813 **/
 814 EFI_STATUS
 815 EFIAPI
 816 NvmExpressGetNamespace (
 817   IN     EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL          *This,
 818   IN     EFI_DEVICE_PATH_PROTOCOL                    *DevicePath,
 819      OUT UINT32                                      *NamespaceId
 820   )
 821 {
 822   NVME_NAMESPACE_DEVICE_PATH       *Node;
 823
 824   if ((This == NULL) || (DevicePath == NULL) || (NamespaceId == NULL)) {
 825     return EFI_INVALID_PARAMETER;
 826   }
 827
 828   if (DevicePath->Type != MESSAGING_DEVICE_PATH) {
 829     return EFI_UNSUPPORTED;
 830   }
 831
 832   Node = (NVME_NAMESPACE_DEVICE_PATH *)DevicePath;
 833
 834   if (DevicePath->SubType == MSG_NVME_NAMESPACE_DP) {
 835     if (DevicePathNodeLength(DevicePath) != sizeof(NVME_NAMESPACE_DEVICE_PATH)) {
 836       return EFI_NOT_FOUND;
 837     }
 838
 839     *NamespaceId = Node->NamespaceId;
 840
 841     return EFI_SUCCESS;
 842   } else {
 843     return EFI_UNSUPPORTED;
 844   }
 845 }
 846
 847 /**
 848   Used to allocate and build a device path node for an NVM Express namespace on an NVM Express controller.
 849
 850   The EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL.BuildDevicePath() function allocates and builds a single device
 851   path node for the NVM Express namespace specified by NamespaceId.
 852
 853   If the NamespaceId is not valid, then EFI_NOT_FOUND is returned.
 854
 855   If DevicePath is NULL, then EFI_INVALID_PARAMETER is returned.
 856
 857   If there are not enough resources to allocate the device path node, then EFI_OUT_OF_RESOURCES is returned.
 858
 859   Otherwise, DevicePath is allocated with the boot service AllocatePool(), the contents of DevicePath are
 860   initialized to describe the NVM Express namespace specified by NamespaceId, and EFI_SUCCESS is returned.
 861
 862   @param[in]     This                A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
 863   @param[in]     NamespaceId         The NVM Express namespace ID  for which a device path node is to be
 864                                      allocated and built. Caller must set the NamespaceId to zero if the
 865                                      device path node will contain a valid UUID.
 866   @param[in,out] DevicePath          A pointer to a single device path node that describes the NVM Express
 867                                      namespace specified by NamespaceId. This function is responsible for
 868                                      allocating the buffer DevicePath with the boot service AllocatePool().
 869                                      It is the caller's responsibility to free DevicePath when the caller
 870                                      is finished with DevicePath.
 871   @retval EFI_SUCCESS                The device path node that describes the NVM Express namespace specified
 872                                      by NamespaceId was allocated and returned in DevicePath.
 873   @retval EFI_NOT_FOUND              The NamespaceId is not valid.
 874   @retval EFI_INVALID_PARAMETER      DevicePath is NULL.
 875   @retval EFI_OUT_OF_RESOURCES       There are not enough resources to allocate the DevicePath node.
 876
 877 **/
 878 EFI_STATUS
 879 EFIAPI
 880 NvmExpressBuildDevicePath (
 881   IN     EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL          *This,
 882   IN     UINT32                                      NamespaceId,
 883   IN OUT EFI_DEVICE_PATH_PROTOCOL                    **DevicePath
 884   )
 885 {
 886   NVME_NAMESPACE_DEVICE_PATH     *Node;
 887   NVME_CONTROLLER_PRIVATE_DATA   *Private;
 888   EFI_STATUS                     Status;
 889   NVME_ADMIN_NAMESPACE_DATA      *NamespaceData;
 890
 891   //
 892   // Validate parameters
 893   //
 894   if ((This == NULL) || (DevicePath == NULL)) {
 895     return EFI_INVALID_PARAMETER;
 896   }
 897
 898   Status  = EFI_SUCCESS;
 899   Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
 900
 901   //
 902   // Check NamespaceId is valid or not.
 903   //
 904   if ((NamespaceId == 0) ||
 905     (NamespaceId > Private->ControllerData->Nn)) {
 906     return EFI_NOT_FOUND;
 907   }
 908
 909   Node = (NVME_NAMESPACE_DEVICE_PATH *)AllocateZeroPool (sizeof (NVME_NAMESPACE_DEVICE_PATH));
 910   if (Node == NULL) {
 911     return EFI_OUT_OF_RESOURCES;
 912   }
 913
 914   Node->Header.Type    = MESSAGING_DEVICE_PATH;
 915   Node->Header.SubType = MSG_NVME_NAMESPACE_DP;
 916   SetDevicePathNodeLength (&Node->Header, sizeof (NVME_NAMESPACE_DEVICE_PATH));
 917   Node->NamespaceId    = NamespaceId;
 918
 919   //
 920   // Allocate a buffer for Identify Namespace data.
 921   //
 922   NamespaceData = NULL;
 923   NamespaceData = AllocateZeroPool(sizeof (NVME_ADMIN_NAMESPACE_DATA));
 924   if(NamespaceData == NULL) {
 925     Status = EFI_OUT_OF_RESOURCES;
 926     goto Exit;
 927   }
 928
 929   //
 930   // Get UUID from specified Identify Namespace data.
 931   //
 932   Status = NvmeIdentifyNamespace (
 933              Private,
 934              NamespaceId,
 935              (VOID *)NamespaceData
 936              );
 937
 938   if (EFI_ERROR(Status)) {
 939     goto Exit;
 940   }
 941
 942   Node->NamespaceUuid = NamespaceData->Eui64;
 943
 944   *DevicePath = (EFI_DEVICE_PATH_PROTOCOL *)Node;
 945
 946 Exit:
 947   if(NamespaceData != NULL) {
 948     FreePool (NamespaceData);
 949   }
 950
 951   if (EFI_ERROR (Status)) {
 952     FreePool (Node);
 953   }
 954
 955   return Status;
 956 }
 957