MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressPassthru.c

   1 /** @file
   2   NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows
   3   NVM Express specification.
   4
   5   (C) Copyright 2014 Hewlett-Packard Development Company, L.P.<BR>
   6   Copyright (c) 2013 - 2016, Intel Corporation. All rights reserved.<BR>
   7   This program and the accompanying materials
   8   are licensed and made available under the terms and conditions of the BSD License
   9   which accompanies this distribution.  The full text of the license may be found at
  10   http://opensource.org/licenses/bsd-license.php.
  11
  12   THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  13   WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  14
  15 **/
  16
  17 #include "NvmExpress.h"
  18
  19 /**
  20   Dump the execution status from a given completion queue entry.
  21
  22   @param[in]     Cq               A pointer to the NVME_CQ item.
  23
  24 **/
  25 VOID
  26 NvmeDumpStatus (
  27   IN NVME_CQ             *Cq
  28   )
  29 {
  30   DEBUG ((EFI_D_VERBOSE, "Dump NVMe Completion Entry Status from [0x%x]:\n", Cq));
  31
  32   DEBUG ((EFI_D_VERBOSE, "  SQ Identifier : [0x%x], Phase Tag : [%d], Cmd Identifier : [0x%x]\n", Cq->Sqid, Cq->Pt, Cq->Cid));
  33
  34   DEBUG ((EFI_D_VERBOSE, "  NVMe Cmd Execution Result - "));
  35
  36   switch (Cq->Sct) {
  37     case 0x0:
  38       switch (Cq->Sc) {
  39         case 0x0:
  40           DEBUG ((EFI_D_VERBOSE, "Successful Completion\n"));
  41           break;
  42         case 0x1:
  43           DEBUG ((EFI_D_VERBOSE, "Invalid Command Opcode\n"));
  44           break;
  45         case 0x2:
  46           DEBUG ((EFI_D_VERBOSE, "Invalid Field in Command\n"));
  47           break;
  48         case 0x3:
  49           DEBUG ((EFI_D_VERBOSE, "Command ID Conflict\n"));
  50           break;
  51         case 0x4:
  52           DEBUG ((EFI_D_VERBOSE, "Data Transfer Error\n"));
  53           break;
  54         case 0x5:
  55           DEBUG ((EFI_D_VERBOSE, "Commands Aborted due to Power Loss Notification\n"));
  56           break;
  57         case 0x6:
  58           DEBUG ((EFI_D_VERBOSE, "Internal Device Error\n"));
  59           break;
  60         case 0x7:
  61           DEBUG ((EFI_D_VERBOSE, "Command Abort Requested\n"));
  62           break;
  63         case 0x8:
  64           DEBUG ((EFI_D_VERBOSE, "Command Aborted due to SQ Deletion\n"));
  65           break;
  66         case 0x9:
  67           DEBUG ((EFI_D_VERBOSE, "Command Aborted due to Failed Fused Command\n"));
  68           break;
  69         case 0xA:
  70           DEBUG ((EFI_D_VERBOSE, "Command Aborted due to Missing Fused Command\n"));
  71           break;
  72         case 0xB:
  73           DEBUG ((EFI_D_VERBOSE, "Invalid Namespace or Format\n"));
  74           break;
  75         case 0xC:
  76           DEBUG ((EFI_D_VERBOSE, "Command Sequence Error\n"));
  77           break;
  78         case 0xD:
  79           DEBUG ((EFI_D_VERBOSE, "Invalid SGL Last Segment Descriptor\n"));
  80           break;
  81         case 0xE:
  82           DEBUG ((EFI_D_VERBOSE, "Invalid Number of SGL Descriptors\n"));
  83           break;
  84         case 0xF:
  85           DEBUG ((EFI_D_VERBOSE, "Data SGL Length Invalid\n"));
  86           break;
  87         case 0x10:
  88           DEBUG ((EFI_D_VERBOSE, "Metadata SGL Length Invalid\n"));
  89           break;
  90         case 0x11:
  91           DEBUG ((EFI_D_VERBOSE, "SGL Descriptor Type Invalid\n"));
  92           break;
  93         case 0x80:
  94           DEBUG ((EFI_D_VERBOSE, "LBA Out of Range\n"));
  95           break;
  96         case 0x81:
  97           DEBUG ((EFI_D_VERBOSE, "Capacity Exceeded\n"));
  98           break;
  99         case 0x82:
 100           DEBUG ((EFI_D_VERBOSE, "Namespace Not Ready\n"));
 101           break;
 102         case 0x83:
 103           DEBUG ((EFI_D_VERBOSE, "Reservation Conflict\n"));
 104           break;
 105       }
 106       break;
 107
 108     case 0x1:
 109       switch (Cq->Sc) {
 110         case 0x0:
 111           DEBUG ((EFI_D_VERBOSE, "Completion Queue Invalid\n"));
 112           break;
 113         case 0x1:
 114           DEBUG ((EFI_D_VERBOSE, "Invalid Queue Identifier\n"));
 115           break;
 116         case 0x2:
 117           DEBUG ((EFI_D_VERBOSE, "Maximum Queue Size Exceeded\n"));
 118           break;
 119         case 0x3:
 120           DEBUG ((EFI_D_VERBOSE, "Abort Command Limit Exceeded\n"));
 121           break;
 122         case 0x5:
 123           DEBUG ((EFI_D_VERBOSE, "Asynchronous Event Request Limit Exceeded\n"));
 124           break;
 125         case 0x6:
 126           DEBUG ((EFI_D_VERBOSE, "Invalid Firmware Slot\n"));
 127           break;
 128         case 0x7:
 129           DEBUG ((EFI_D_VERBOSE, "Invalid Firmware Image\n"));
 130           break;
 131         case 0x8:
 132           DEBUG ((EFI_D_VERBOSE, "Invalid Interrupt Vector\n"));
 133           break;
 134         case 0x9:
 135           DEBUG ((EFI_D_VERBOSE, "Invalid Log Page\n"));
 136           break;
 137         case 0xA:
 138           DEBUG ((EFI_D_VERBOSE, "Invalid Format\n"));
 139           break;
 140         case 0xB:
 141           DEBUG ((EFI_D_VERBOSE, "Firmware Application Requires Conventional Reset\n"));
 142           break;
 143         case 0xC:
 144           DEBUG ((EFI_D_VERBOSE, "Invalid Queue Deletion\n"));
 145           break;
 146         case 0xD:
 147           DEBUG ((EFI_D_VERBOSE, "Feature Identifier Not Saveable\n"));
 148           break;
 149         case 0xE:
 150           DEBUG ((EFI_D_VERBOSE, "Feature Not Changeable\n"));
 151           break;
 152         case 0xF:
 153           DEBUG ((EFI_D_VERBOSE, "Feature Not Namespace Specific\n"));
 154           break;
 155         case 0x10:
 156           DEBUG ((EFI_D_VERBOSE, "Firmware Application Requires NVM Subsystem Reset\n"));
 157           break;
 158         case 0x80:
 159           DEBUG ((EFI_D_VERBOSE, "Conflicting Attributes\n"));
 160           break;
 161         case 0x81:
 162           DEBUG ((EFI_D_VERBOSE, "Invalid Protection Information\n"));
 163           break;
 164         case 0x82:
 165           DEBUG ((EFI_D_VERBOSE, "Attempted Write to Read Only Range\n"));
 166           break;
 167       }
 168       break;
 169
 170     case 0x2:
 171       switch (Cq->Sc) {
 172         case 0x80:
 173           DEBUG ((EFI_D_VERBOSE, "Write Fault\n"));
 174           break;
 175         case 0x81:
 176           DEBUG ((EFI_D_VERBOSE, "Unrecovered Read Error\n"));
 177           break;
 178         case 0x82:
 179           DEBUG ((EFI_D_VERBOSE, "End-to-end Guard Check Error\n"));
 180           break;
 181         case 0x83:
 182           DEBUG ((EFI_D_VERBOSE, "End-to-end Application Tag Check Error\n"));
 183           break;
 184         case 0x84:
 185           DEBUG ((EFI_D_VERBOSE, "End-to-end Reference Tag Check Error\n"));
 186           break;
 187         case 0x85:
 188           DEBUG ((EFI_D_VERBOSE, "Compare Failure\n"));
 189           break;
 190         case 0x86:
 191           DEBUG ((EFI_D_VERBOSE, "Access Denied\n"));
 192           break;
 193       }
 194       break;
 195
 196     default:
 197       break;
 198   }
 199 }
 200
 201 /**
 202   Create PRP lists for data transfer which is larger than 2 memory pages.
 203   Note here we calcuate the number of required PRP lists and allocate them at one time.
 204
 205   @param[in]     PciIo               A pointer to the EFI_PCI_IO_PROTOCOL instance.
 206   @param[in]     PhysicalAddr        The physical base address of data buffer.
 207   @param[in]     Pages               The number of pages to be transfered.
 208   @param[out]    PrpListHost         The host base address of PRP lists.
 209   @param[in,out] PrpListNo           The number of PRP List.
 210   @param[out]    Mapping             The mapping value returned from PciIo.Map().
 211
 212   @retval The pointer to the first PRP List of the PRP lists.
 213
 214 **/
 215 VOID*
 216 NvmeCreatePrpList (
 217   IN     EFI_PCI_IO_PROTOCOL          *PciIo,
 218   IN     EFI_PHYSICAL_ADDRESS         PhysicalAddr,
 219   IN     UINTN                        Pages,
 220      OUT VOID                         **PrpListHost,
 221   IN OUT UINTN                        *PrpListNo,
 222      OUT VOID                         **Mapping
 223   )
 224 {
 225   UINTN                       PrpEntryNo;
 226   UINT64                      PrpListBase;
 227   UINTN                       PrpListIndex;
 228   UINTN                       PrpEntryIndex;
 229   UINT64                      Remainder;
 230   EFI_PHYSICAL_ADDRESS        PrpListPhyAddr;
 231   UINTN                       Bytes;
 232   EFI_STATUS                  Status;
 233
 234   //
 235   // The number of Prp Entry in a memory page.
 236   //
 237   PrpEntryNo = EFI_PAGE_SIZE / sizeof (UINT64);
 238
 239   //
 240   // Calculate total PrpList number.
 241   //
 242   *PrpListNo = (UINTN)DivU64x64Remainder ((UINT64)Pages, (UINT64)PrpEntryNo - 1, &Remainder);
 243   if (*PrpListNo == 0) {
 244     *PrpListNo = 1;
 245   } else if ((Remainder != 0) && (Remainder != 1)) {
 246     *PrpListNo += 1;
 247   } else if (Remainder == 1) {
 248     Remainder = PrpEntryNo;
 249   } else if (Remainder == 0) {
 250     Remainder = PrpEntryNo - 1;
 251   }
 252
 253   Status = PciIo->AllocateBuffer (
 254                     PciIo,
 255                     AllocateAnyPages,
 256                     EfiBootServicesData,
 257                     *PrpListNo,
 258                     PrpListHost,
 259                     0
 260                     );
 261
 262   if (EFI_ERROR (Status)) {
 263     return NULL;
 264   }
 265
 266   Bytes = EFI_PAGES_TO_SIZE (*PrpListNo);
 267   Status = PciIo->Map (
 268                     PciIo,
 269                     EfiPciIoOperationBusMasterCommonBuffer,
 270                     *PrpListHost,
 271                     &Bytes,
 272                     &PrpListPhyAddr,
 273                     Mapping
 274                     );
 275
 276   if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (*PrpListNo))) {
 277     DEBUG ((EFI_D_ERROR, "NvmeCreatePrpList: create PrpList failure!\n"));
 278     goto EXIT;
 279   }
 280   //
 281   // Fill all PRP lists except of last one.
 282   //
 283   ZeroMem (*PrpListHost, Bytes);
 284   for (PrpListIndex = 0; PrpListIndex < *PrpListNo - 1; ++PrpListIndex) {
 285     PrpListBase = *(UINT64*)PrpListHost + PrpListIndex * EFI_PAGE_SIZE;
 286
 287     for (PrpEntryIndex = 0; PrpEntryIndex < PrpEntryNo; ++PrpEntryIndex) {
 288       if (PrpEntryIndex != PrpEntryNo - 1) {
 289         //
 290         // Fill all PRP entries except of last one.
 291         //
 292         *((UINT64*)(UINTN)PrpListBase + PrpEntryIndex) = PhysicalAddr;
 293         PhysicalAddr += EFI_PAGE_SIZE;
 294       } else {
 295         //
 296         // Fill last PRP entries with next PRP List pointer.
 297         //
 298         *((UINT64*)(UINTN)PrpListBase + PrpEntryIndex) = PrpListPhyAddr + (PrpListIndex + 1) * EFI_PAGE_SIZE;
 299       }
 300     }
 301   }
 302   //
 303   // Fill last PRP list.
 304   //
 305   PrpListBase = *(UINT64*)PrpListHost + PrpListIndex * EFI_PAGE_SIZE;
 306   for (PrpEntryIndex = 0; PrpEntryIndex < Remainder; ++PrpEntryIndex) {
 307     *((UINT64*)(UINTN)PrpListBase + PrpEntryIndex) = PhysicalAddr;
 308     PhysicalAddr += EFI_PAGE_SIZE;
 309   }
 310
 311   return (VOID*)(UINTN)PrpListPhyAddr;
 312
 313 EXIT:
 314   PciIo->FreeBuffer (PciIo, *PrpListNo, *PrpListHost);
 315   return NULL;
 316 }
 317
 318
 319 /**
 320   Sends an NVM Express Command Packet to an NVM Express controller or namespace. This function supports
 321   both blocking I/O and non-blocking I/O. The blocking I/O functionality is required, and the non-blocking
 322   I/O functionality is optional.
 323
 324
 325   @param[in]     This                A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
 326   @param[in]     NamespaceId         A 32 bit namespace ID as defined in the NVMe specification to which the NVM Express Command
 327                                      Packet will be sent.  A value of 0 denotes the NVM Express controller, a value of all 0xFF's
 328                                      (all bytes are 0xFF) in the namespace ID specifies that the command packet should be sent to
 329                                      all valid namespaces.
 330   @param[in,out] Packet              A pointer to the NVM Express Command Packet.
 331   @param[in]     Event               If non-blocking I/O is not supported then Event is ignored, and blocking I/O is performed.
 332                                      If Event is NULL, then blocking I/O is performed. If Event is not NULL and non-blocking I/O
 333                                      is supported, then non-blocking I/O is performed, and Event will be signaled when the NVM
 334                                      Express Command Packet completes.
 335
 336   @retval EFI_SUCCESS                The NVM Express Command Packet was sent by the host. TransferLength bytes were transferred
 337                                      to, or from DataBuffer.
 338   @retval EFI_BAD_BUFFER_SIZE        The NVM Express Command Packet was not executed. The number of bytes that could be transferred
 339                                      is returned in TransferLength.
 340   @retval EFI_NOT_READY              The NVM Express Command Packet could not be sent because the controller is not ready. The caller
 341                                      may retry again later.
 342   @retval EFI_DEVICE_ERROR           A device error occurred while attempting to send the NVM Express Command Packet.
 343   @retval EFI_INVALID_PARAMETER      NamespaceId or the contents of EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET are invalid. The NVM
 344                                      Express Command Packet was not sent, so no additional status information is available.
 345   @retval EFI_UNSUPPORTED            The command described by the NVM Express Command Packet is not supported by the NVM Express
 346                                      controller. The NVM Express Command Packet was not sent so no additional status information
 347                                      is available.
 348   @retval EFI_TIMEOUT                A timeout occurred while waiting for the NVM Express Command Packet to execute.
 349
 350 **/
 351 EFI_STATUS
 352 EFIAPI
 353 NvmExpressPassThru (
 354   IN     EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL          *This,
 355   IN     UINT32                                      NamespaceId,
 356   IN OUT EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET    *Packet,
 357   IN     EFI_EVENT                                   Event OPTIONAL
 358   )
 359 {
 360   NVME_CONTROLLER_PRIVATE_DATA   *Private;
 361   EFI_STATUS                     Status;
 362   EFI_PCI_IO_PROTOCOL            *PciIo;
 363   NVME_SQ                        *Sq;
 364   NVME_CQ                        *Cq;
 365   UINT16                         QueueId;
 366   UINT32                         Bytes;
 367   UINT16                         Offset;
 368   EFI_EVENT                      TimerEvent;
 369   EFI_PCI_IO_PROTOCOL_OPERATION  Flag;
 370   EFI_PHYSICAL_ADDRESS           PhyAddr;
 371   VOID                           *MapData;
 372   VOID                           *MapMeta;
 373   VOID                           *MapPrpList;
 374   UINTN                          MapLength;
 375   UINT64                         *Prp;
 376   VOID                           *PrpListHost;
 377   UINTN                          PrpListNo;
 378   UINT32                         Attributes;
 379   UINT32                         IoAlign;
 380   UINT32                         MaxTransLen;
 381   UINT32                         Data;
 382   NVME_PASS_THRU_ASYNC_REQ       *AsyncRequest;
 383   EFI_TPL                        OldTpl;
 384
 385   //
 386   // check the data fields in Packet parameter.
 387   //
 388   if ((This == NULL) || (Packet == NULL)) {
 389     return EFI_INVALID_PARAMETER;
 390   }
 391
 392   if ((Packet->NvmeCmd == NULL) || (Packet->NvmeCompletion == NULL)) {
 393     return EFI_INVALID_PARAMETER;
 394   }
 395
 396   if (Packet->QueueType != NVME_ADMIN_QUEUE && Packet->QueueType != NVME_IO_QUEUE) {
 397     return EFI_INVALID_PARAMETER;
 398   }
 399
 400   //
 401   // 'Attributes' with neither EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL nor
 402   // EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL set is an illegal
 403   // configuration.
 404   //
 405   Attributes  = This->Mode->Attributes;
 406   if ((Attributes & (EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL |
 407     EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL)) == 0) {
 408     return EFI_INVALID_PARAMETER;
 409   }
 410
 411   //
 412   // Buffer alignment check for TransferBuffer & MetadataBuffer.
 413   //
 414   IoAlign     = This->Mode->IoAlign;
 415   if (IoAlign > 0 && (((UINTN) Packet->TransferBuffer & (IoAlign - 1)) != 0)) {
 416     return EFI_INVALID_PARAMETER;
 417   }
 418
 419   if (IoAlign > 0 && (((UINTN) Packet->MetadataBuffer & (IoAlign - 1)) != 0)) {
 420     return EFI_INVALID_PARAMETER;
 421   }
 422
 423   Private     = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
 424
 425   //
 426   // Check NamespaceId is valid or not.
 427   //
 428   if ((NamespaceId > Private->ControllerData->Nn) &&
 429       (NamespaceId != (UINT32) -1)) {
 430     return EFI_INVALID_PARAMETER;
 431   }
 432
 433   //
 434   // Check whether TransferLength exceeds the maximum data transfer size.
 435   //
 436   if (Private->ControllerData->Mdts != 0) {
 437     MaxTransLen = (1 << (Private->ControllerData->Mdts)) *
 438                   (1 << (Private->Cap.Mpsmin + 12));
 439     if (Packet->TransferLength > MaxTransLen) {
 440       Packet->TransferLength = MaxTransLen;
 441       return EFI_BAD_BUFFER_SIZE;
 442     }
 443   }
 444
 445   PciIo       = Private->PciIo;
 446   MapData     = NULL;
 447   MapMeta     = NULL;
 448   MapPrpList  = NULL;
 449   PrpListHost = NULL;
 450   PrpListNo   = 0;
 451   Prp         = NULL;
 452   TimerEvent  = NULL;
 453   Status      = EFI_SUCCESS;
 454
 455   if (Packet->QueueType == NVME_ADMIN_QUEUE) {
 456     QueueId = 0;
 457   } else {
 458     if (Event == NULL) {
 459       QueueId = 1;
 460     } else {
 461       QueueId = 2;
 462
 463       //
 464       // Submission queue full check.
 465       //
 466       if ((Private->SqTdbl[QueueId].Sqt + 1) % (NVME_ASYNC_CSQ_SIZE + 1) ==
 467           Private->AsyncSqHead) {
 468         return EFI_NOT_READY;
 469       }
 470     }
 471   }
 472   Sq  = Private->SqBuffer[QueueId] + Private->SqTdbl[QueueId].Sqt;
 473   Cq  = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
 474
 475   if (Packet->NvmeCmd->Nsid != NamespaceId) {
 476     return EFI_INVALID_PARAMETER;
 477   }
 478
 479   ZeroMem (Sq, sizeof (NVME_SQ));
 480   Sq->Opc  = (UINT8)Packet->NvmeCmd->Cdw0.Opcode;
 481   Sq->Fuse = (UINT8)Packet->NvmeCmd->Cdw0.FusedOperation;
 482   Sq->Cid  = Private->Cid[QueueId]++;
 483   Sq->Nsid = Packet->NvmeCmd->Nsid;
 484
 485   //
 486   // Currently we only support PRP for data transfer, SGL is NOT supported.
 487   //
 488   ASSERT (Sq->Psdt == 0);
 489   if (Sq->Psdt != 0) {
 490     DEBUG ((EFI_D_ERROR, "NvmExpressPassThru: doesn't support SGL mechanism\n"));
 491     return EFI_UNSUPPORTED;
 492   }
 493
 494   Sq->Prp[0] = (UINT64)(UINTN)Packet->TransferBuffer;
 495   //
 496   // If the NVMe cmd has data in or out, then mapping the user buffer to the PCI controller specific addresses.
 497   // Note here we don't handle data buffer for CreateIOSubmitionQueue and CreateIOCompletionQueue cmds because
 498   // these two cmds are special which requires their data buffer must support simultaneous access by both the
 499   // processor and a PCI Bus Master. It's caller's responsbility to ensure this.
 500   //
 501   if (((Sq->Opc & (BIT0 | BIT1)) != 0) && (Sq->Opc != NVME_ADMIN_CRIOCQ_CMD) && (Sq->Opc != NVME_ADMIN_CRIOSQ_CMD)) {
 502     if ((Packet->TransferLength == 0) || (Packet->TransferBuffer == NULL)) {
 503       return EFI_INVALID_PARAMETER;
 504     }
 505
 506     if ((Sq->Opc & BIT0) != 0) {
 507       Flag = EfiPciIoOperationBusMasterRead;
 508     } else {
 509       Flag = EfiPciIoOperationBusMasterWrite;
 510     }
 511
 512     MapLength = Packet->TransferLength;
 513     Status = PciIo->Map (
 514                       PciIo,
 515                       Flag,
 516                       Packet->TransferBuffer,
 517                       &MapLength,
 518                       &PhyAddr,
 519                       &MapData
 520                       );
 521     if (EFI_ERROR (Status) || (Packet->TransferLength != MapLength)) {
 522       return EFI_OUT_OF_RESOURCES;
 523     }
 524
 525     Sq->Prp[0] = PhyAddr;
 526     Sq->Prp[1] = 0;
 527
 528     if((Packet->MetadataLength != 0) && (Packet->MetadataBuffer != NULL)) {
 529       MapLength = Packet->MetadataLength;
 530       Status = PciIo->Map (
 531                         PciIo,
 532                         Flag,
 533                         Packet->MetadataBuffer,
 534                         &MapLength,
 535                         &PhyAddr,
 536                         &MapMeta
 537                         );
 538       if (EFI_ERROR (Status) || (Packet->MetadataLength != MapLength)) {
 539         PciIo->Unmap (
 540                  PciIo,
 541                  MapData
 542                  );
 543
 544         return EFI_OUT_OF_RESOURCES;
 545       }
 546       Sq->Mptr = PhyAddr;
 547     }
 548   }
 549   //
 550   // If the buffer size spans more than two memory pages (page size as defined in CC.Mps),
 551   // then build a PRP list in the second PRP submission queue entry.
 552   //
 553   Offset = ((UINT16)Sq->Prp[0]) & (EFI_PAGE_SIZE - 1);
 554   Bytes  = Packet->TransferLength;
 555
 556   if ((Offset + Bytes) > (EFI_PAGE_SIZE * 2)) {
 557     //
 558     // Create PrpList for remaining data buffer.
 559     //
 560     PhyAddr = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
 561     Prp = NvmeCreatePrpList (PciIo, PhyAddr, EFI_SIZE_TO_PAGES(Offset + Bytes) - 1, &PrpListHost, &PrpListNo, &MapPrpList);
 562     if (Prp == NULL) {
 563       goto EXIT;
 564     }
 565
 566     Sq->Prp[1] = (UINT64)(UINTN)Prp;
 567   } else if ((Offset + Bytes) > EFI_PAGE_SIZE) {
 568     Sq->Prp[1] = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
 569   }
 570
 571   if(Packet->NvmeCmd->Flags & CDW2_VALID) {
 572     Sq->Rsvd2 = (UINT64)Packet->NvmeCmd->Cdw2;
 573   }
 574   if(Packet->NvmeCmd->Flags & CDW3_VALID) {
 575     Sq->Rsvd2 |= LShiftU64 ((UINT64)Packet->NvmeCmd->Cdw3, 32);
 576   }
 577   if(Packet->NvmeCmd->Flags & CDW10_VALID) {
 578     Sq->Payload.Raw.Cdw10 = Packet->NvmeCmd->Cdw10;
 579   }
 580   if(Packet->NvmeCmd->Flags & CDW11_VALID) {
 581     Sq->Payload.Raw.Cdw11 = Packet->NvmeCmd->Cdw11;
 582   }
 583   if(Packet->NvmeCmd->Flags & CDW12_VALID) {
 584     Sq->Payload.Raw.Cdw12 = Packet->NvmeCmd->Cdw12;
 585   }
 586   if(Packet->NvmeCmd->Flags & CDW13_VALID) {
 587     Sq->Payload.Raw.Cdw13 = Packet->NvmeCmd->Cdw13;
 588   }
 589   if(Packet->NvmeCmd->Flags & CDW14_VALID) {
 590     Sq->Payload.Raw.Cdw14 = Packet->NvmeCmd->Cdw14;
 591   }
 592   if(Packet->NvmeCmd->Flags & CDW15_VALID) {
 593     Sq->Payload.Raw.Cdw15 = Packet->NvmeCmd->Cdw15;
 594   }
 595
 596   //
 597   // Ring the submission queue doorbell.
 598   //
 599   if ((Event != NULL) && (QueueId != 0)) {
 600     Private->SqTdbl[QueueId].Sqt =
 601       (Private->SqTdbl[QueueId].Sqt + 1) % (NVME_ASYNC_CSQ_SIZE + 1);
 602   } else {
 603     Private->SqTdbl[QueueId].Sqt ^= 1;
 604   }
 605   Data = ReadUnaligned32 ((UINT32*)&Private->SqTdbl[QueueId]);
 606   PciIo->Mem.Write (
 607                PciIo,
 608                EfiPciIoWidthUint32,
 609                NVME_BAR,
 610                NVME_SQTDBL_OFFSET(QueueId, Private->Cap.Dstrd),
 611                1,
 612                &Data
 613                );
 614
 615   //
 616   // For non-blocking requests, return directly if the command is placed
 617   // in the submission queue.
 618   //
 619   if ((Event != NULL) && (QueueId != 0)) {
 620     AsyncRequest = AllocateZeroPool (sizeof (NVME_PASS_THRU_ASYNC_REQ));
 621     if (AsyncRequest == NULL) {
 622       Status = EFI_DEVICE_ERROR;
 623       goto EXIT;
 624     }
 625
 626     AsyncRequest->Signature     = NVME_PASS_THRU_ASYNC_REQ_SIG;
 627     AsyncRequest->Packet        = Packet;
 628     AsyncRequest->CommandId     = Sq->Cid;
 629     AsyncRequest->CallerEvent   = Event;
 630     AsyncRequest->MapData       = MapData;
 631     AsyncRequest->MapMeta       = MapMeta;
 632     AsyncRequest->MapPrpList    = MapPrpList;
 633     AsyncRequest->PrpListNo     = PrpListNo;
 634     AsyncRequest->PrpListHost   = PrpListHost;
 635
 636     OldTpl = gBS->RaiseTPL (TPL_NOTIFY);
 637     InsertTailList (&Private->AsyncPassThruQueue, &AsyncRequest->Link);
 638     gBS->RestoreTPL (OldTpl);
 639
 640     return EFI_SUCCESS;
 641   }
 642
 643   Status = gBS->CreateEvent (
 644                   EVT_TIMER,
 645                   TPL_CALLBACK,
 646                   NULL,
 647                   NULL,
 648                   &TimerEvent
 649                   );
 650   if (EFI_ERROR (Status)) {
 651     goto EXIT;
 652   }
 653
 654   Status = gBS->SetTimer(TimerEvent, TimerRelative, Packet->CommandTimeout);
 655
 656   if (EFI_ERROR(Status)) {
 657     goto EXIT;
 658   }
 659
 660   //
 661   // Wait for completion queue to get filled in.
 662   //
 663   Status = EFI_TIMEOUT;
 664   while (EFI_ERROR (gBS->CheckEvent (TimerEvent))) {
 665     if (Cq->Pt != Private->Pt[QueueId]) {
 666       Status = EFI_SUCCESS;
 667       break;
 668     }
 669   }
 670
 671   //
 672   // Check the NVMe cmd execution result
 673   //
 674   if (Status != EFI_TIMEOUT) {
 675     if ((Cq->Sct == 0) && (Cq->Sc == 0)) {
 676       Status = EFI_SUCCESS;
 677     } else {
 678       Status = EFI_DEVICE_ERROR;
 679       //
 680       // Copy the Respose Queue entry for this command to the callers response buffer
 681       //
 682       CopyMem(Packet->NvmeCompletion, Cq, sizeof(EFI_NVM_EXPRESS_COMPLETION));
 683
 684       //
 685       // Dump every completion entry status for debugging.
 686       //
 687       DEBUG_CODE_BEGIN();
 688         NvmeDumpStatus(Cq);
 689       DEBUG_CODE_END();
 690     }
 691   }
 692
 693   if ((Private->CqHdbl[QueueId].Cqh ^= 1) == 0) {
 694     Private->Pt[QueueId] ^= 1;
 695   }
 696
 697   Data = ReadUnaligned32 ((UINT32*)&Private->CqHdbl[QueueId]);
 698   PciIo->Mem.Write (
 699                PciIo,
 700                EfiPciIoWidthUint32,
 701                NVME_BAR,
 702                NVME_CQHDBL_OFFSET(QueueId, Private->Cap.Dstrd),
 703                1,
 704                &Data
 705                );
 706
 707   //
 708   // For now, the code does not support the non-blocking feature for admin queue.
 709   // If Event is not NULL for admin queue, signal the caller's event here.
 710   //
 711   if (Event != NULL) {
 712     ASSERT (QueueId == 0);
 713     gBS->SignalEvent (Event);
 714   }
 715
 716 EXIT:
 717   if (MapData != NULL) {
 718     PciIo->Unmap (
 719              PciIo,
 720              MapData
 721              );
 722   }
 723
 724   if (MapMeta != NULL) {
 725     PciIo->Unmap (
 726              PciIo,
 727              MapMeta
 728              );
 729   }
 730
 731   if (MapPrpList != NULL) {
 732     PciIo->Unmap (
 733              PciIo,
 734              MapPrpList
 735              );
 736   }
 737
 738   if (Prp != NULL) {
 739     PciIo->FreeBuffer (PciIo, PrpListNo, PrpListHost);
 740   }
 741
 742   if (TimerEvent != NULL) {
 743     gBS->CloseEvent (TimerEvent);
 744   }
 745   return Status;
 746 }
 747
 748 /**
 749   Used to retrieve the next namespace ID for this NVM Express controller.
 750
 751   The EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL.GetNextNamespace() function retrieves the next valid
 752   namespace ID on this NVM Express controller.
 753
 754   If on input the value pointed to by NamespaceId is 0xFFFFFFFF, then the first valid namespace
 755   ID defined on the NVM Express controller is returned in the location pointed to by NamespaceId
 756   and a status of EFI_SUCCESS is returned.
 757
 758   If on input the value pointed to by NamespaceId is an invalid namespace ID other than 0xFFFFFFFF,
 759   then EFI_INVALID_PARAMETER is returned.
 760
 761   If on input the value pointed to by NamespaceId is a valid namespace ID, then the next valid
 762   namespace ID on the NVM Express controller is returned in the location pointed to by NamespaceId,
 763   and EFI_SUCCESS is returned.
 764
 765   If the value pointed to by NamespaceId is the namespace ID of the last namespace on the NVM
 766   Express controller, then EFI_NOT_FOUND is returned.
 767
 768   @param[in]     This           A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
 769   @param[in,out] NamespaceId    On input, a pointer to a legal NamespaceId for an NVM Express
 770                                 namespace present on the NVM Express controller. On output, a
 771                                 pointer to the next NamespaceId of an NVM Express namespace on
 772                                 an NVM Express controller. An input value of 0xFFFFFFFF retrieves
 773                                 the first NamespaceId for an NVM Express namespace present on an
 774                                 NVM Express controller.
 775
 776   @retval EFI_SUCCESS           The Namespace ID of the next Namespace was returned.
 777   @retval EFI_NOT_FOUND         There are no more namespaces defined on this controller.
 778   @retval EFI_INVALID_PARAMETER NamespaceId is an invalid value other than 0xFFFFFFFF.
 779
 780 **/
 781 EFI_STATUS
 782 EFIAPI
 783 NvmExpressGetNextNamespace (
 784   IN     EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL          *This,
 785   IN OUT UINT32                                      *NamespaceId
 786   )
 787 {
 788   NVME_CONTROLLER_PRIVATE_DATA     *Private;
 789   NVME_ADMIN_NAMESPACE_DATA        *NamespaceData;
 790   UINT32                           NextNamespaceId;
 791   EFI_STATUS                       Status;
 792
 793   if ((This == NULL) || (NamespaceId == NULL)) {
 794     return EFI_INVALID_PARAMETER;
 795   }
 796
 797   NamespaceData = NULL;
 798   Status        = EFI_NOT_FOUND;
 799
 800   Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
 801   //
 802   // If the NamespaceId input value is 0xFFFFFFFF, then get the first valid namespace ID
 803   //
 804   if (*NamespaceId == 0xFFFFFFFF) {
 805     //
 806     // Start with the first namespace ID
 807     //
 808     NextNamespaceId = 1;
 809     //
 810     // Allocate buffer for Identify Namespace data.
 811     //
 812     NamespaceData = (NVME_ADMIN_NAMESPACE_DATA *)AllocateZeroPool (sizeof (NVME_ADMIN_NAMESPACE_DATA));
 813
 814     if (NamespaceData == NULL) {
 815       return EFI_NOT_FOUND;
 816     }
 817
 818     Status = NvmeIdentifyNamespace (Private, NextNamespaceId, NamespaceData);
 819     if (EFI_ERROR(Status)) {
 820       goto Done;
 821     }
 822
 823     *NamespaceId = NextNamespaceId;
 824   } else {
 825     if (*NamespaceId > Private->ControllerData->Nn) {
 826       return EFI_INVALID_PARAMETER;
 827     }
 828
 829     NextNamespaceId = *NamespaceId + 1;
 830     if (NextNamespaceId > Private->ControllerData->Nn) {
 831       return EFI_NOT_FOUND;
 832     }
 833
 834     //
 835     // Allocate buffer for Identify Namespace data.
 836     //
 837     NamespaceData = (NVME_ADMIN_NAMESPACE_DATA *)AllocateZeroPool (sizeof (NVME_ADMIN_NAMESPACE_DATA));
 838     if (NamespaceData == NULL) {
 839       return EFI_NOT_FOUND;
 840     }
 841
 842     Status = NvmeIdentifyNamespace (Private, NextNamespaceId, NamespaceData);
 843     if (EFI_ERROR(Status)) {
 844       goto Done;
 845     }
 846
 847     *NamespaceId = NextNamespaceId;
 848   }
 849
 850 Done:
 851   if (NamespaceData != NULL) {
 852     FreePool(NamespaceData);
 853   }
 854
 855   return Status;
 856 }
 857
 858 /**
 859   Used to translate a device path node to a namespace ID.
 860
 861   The EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL.GetNamespace() function determines the namespace ID associated with the
 862   namespace described by DevicePath.
 863
 864   If DevicePath is a device path node type that the NVM Express Pass Thru driver supports, then the NVM Express
 865   Pass Thru driver will attempt to translate the contents DevicePath into a namespace ID.
 866
 867   If this translation is successful, then that namespace ID is returned in NamespaceId, and EFI_SUCCESS is returned
 868
 869   @param[in]  This                A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
 870   @param[in]  DevicePath          A pointer to the device path node that describes an NVM Express namespace on
 871                                   the NVM Express controller.
 872   @param[out] NamespaceId         The NVM Express namespace ID contained in the device path node.
 873
 874   @retval EFI_SUCCESS             DevicePath was successfully translated to NamespaceId.
 875   @retval EFI_INVALID_PARAMETER   If DevicePath or NamespaceId are NULL, then EFI_INVALID_PARAMETER is returned.
 876   @retval EFI_UNSUPPORTED         If DevicePath is not a device path node type that the NVM Express Pass Thru driver
 877                                   supports, then EFI_UNSUPPORTED is returned.
 878   @retval EFI_NOT_FOUND           If DevicePath is a device path node type that the NVM Express Pass Thru driver
 879                                   supports, but there is not a valid translation from DevicePath to a namespace ID,
 880                                   then EFI_NOT_FOUND is returned.
 881 **/
 882 EFI_STATUS
 883 EFIAPI
 884 NvmExpressGetNamespace (
 885   IN     EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL          *This,
 886   IN     EFI_DEVICE_PATH_PROTOCOL                    *DevicePath,
 887      OUT UINT32                                      *NamespaceId
 888   )
 889 {
 890   NVME_NAMESPACE_DEVICE_PATH       *Node;
 891   NVME_CONTROLLER_PRIVATE_DATA     *Private;
 892
 893   if ((This == NULL) || (DevicePath == NULL) || (NamespaceId == NULL)) {
 894     return EFI_INVALID_PARAMETER;
 895   }
 896
 897   if (DevicePath->Type != MESSAGING_DEVICE_PATH) {
 898     return EFI_UNSUPPORTED;
 899   }
 900
 901   Node    = (NVME_NAMESPACE_DEVICE_PATH *)DevicePath;
 902   Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
 903
 904   if (DevicePath->SubType == MSG_NVME_NAMESPACE_DP) {
 905     if (DevicePathNodeLength(DevicePath) != sizeof(NVME_NAMESPACE_DEVICE_PATH)) {
 906       return EFI_NOT_FOUND;
 907     }
 908
 909     //
 910     // Check NamespaceId in the device path node is valid or not.
 911     //
 912     if ((Node->NamespaceId == 0) ||
 913       (Node->NamespaceId > Private->ControllerData->Nn)) {
 914       return EFI_NOT_FOUND;
 915     }
 916
 917     *NamespaceId = Node->NamespaceId;
 918
 919     return EFI_SUCCESS;
 920   } else {
 921     return EFI_UNSUPPORTED;
 922   }
 923 }
 924
 925 /**
 926   Used to allocate and build a device path node for an NVM Express namespace on an NVM Express controller.
 927
 928   The EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL.BuildDevicePath() function allocates and builds a single device
 929   path node for the NVM Express namespace specified by NamespaceId.
 930
 931   If the NamespaceId is not valid, then EFI_NOT_FOUND is returned.
 932
 933   If DevicePath is NULL, then EFI_INVALID_PARAMETER is returned.
 934
 935   If there are not enough resources to allocate the device path node, then EFI_OUT_OF_RESOURCES is returned.
 936
 937   Otherwise, DevicePath is allocated with the boot service AllocatePool(), the contents of DevicePath are
 938   initialized to describe the NVM Express namespace specified by NamespaceId, and EFI_SUCCESS is returned.
 939
 940   @param[in]     This                A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
 941   @param[in]     NamespaceId         The NVM Express namespace ID  for which a device path node is to be
 942                                      allocated and built. Caller must set the NamespaceId to zero if the
 943                                      device path node will contain a valid UUID.
 944   @param[in,out] DevicePath          A pointer to a single device path node that describes the NVM Express
 945                                      namespace specified by NamespaceId. This function is responsible for
 946                                      allocating the buffer DevicePath with the boot service AllocatePool().
 947                                      It is the caller's responsibility to free DevicePath when the caller
 948                                      is finished with DevicePath.
 949   @retval EFI_SUCCESS                The device path node that describes the NVM Express namespace specified
 950                                      by NamespaceId was allocated and returned in DevicePath.
 951   @retval EFI_NOT_FOUND              The NamespaceId is not valid.
 952   @retval EFI_INVALID_PARAMETER      DevicePath is NULL.
 953   @retval EFI_OUT_OF_RESOURCES       There are not enough resources to allocate the DevicePath node.
 954
 955 **/
 956 EFI_STATUS
 957 EFIAPI
 958 NvmExpressBuildDevicePath (
 959   IN     EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL          *This,
 960   IN     UINT32                                      NamespaceId,
 961   IN OUT EFI_DEVICE_PATH_PROTOCOL                    **DevicePath
 962   )
 963 {
 964   NVME_NAMESPACE_DEVICE_PATH     *Node;
 965   NVME_CONTROLLER_PRIVATE_DATA   *Private;
 966   EFI_STATUS                     Status;
 967   NVME_ADMIN_NAMESPACE_DATA      *NamespaceData;
 968
 969   //
 970   // Validate parameters
 971   //
 972   if ((This == NULL) || (DevicePath == NULL)) {
 973     return EFI_INVALID_PARAMETER;
 974   }
 975
 976   Status  = EFI_SUCCESS;
 977   Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
 978
 979   //
 980   // Check NamespaceId is valid or not.
 981   //
 982   if ((NamespaceId == 0) ||
 983     (NamespaceId > Private->ControllerData->Nn)) {
 984     return EFI_NOT_FOUND;
 985   }
 986
 987   Node = (NVME_NAMESPACE_DEVICE_PATH *)AllocateZeroPool (sizeof (NVME_NAMESPACE_DEVICE_PATH));
 988   if (Node == NULL) {
 989     return EFI_OUT_OF_RESOURCES;
 990   }
 991
 992   Node->Header.Type    = MESSAGING_DEVICE_PATH;
 993   Node->Header.SubType = MSG_NVME_NAMESPACE_DP;
 994   SetDevicePathNodeLength (&Node->Header, sizeof (NVME_NAMESPACE_DEVICE_PATH));
 995   Node->NamespaceId    = NamespaceId;
 996
 997   //
 998   // Allocate a buffer for Identify Namespace data.
 999   //
1000   NamespaceData = NULL;
1001   NamespaceData = AllocateZeroPool(sizeof (NVME_ADMIN_NAMESPACE_DATA));
1002   if(NamespaceData == NULL) {
1003     Status = EFI_OUT_OF_RESOURCES;
1004     goto Exit;
1005   }
1006
1007   //
1008   // Get UUID from specified Identify Namespace data.
1009   //
1010   Status = NvmeIdentifyNamespace (
1011              Private,
1012              NamespaceId,
1013              (VOID *)NamespaceData
1014              );
1015
1016   if (EFI_ERROR(Status)) {
1017     goto Exit;
1018   }
1019
1020   Node->NamespaceUuid = NamespaceData->Eui64;
1021
1022   *DevicePath = (EFI_DEVICE_PATH_PROTOCOL *)Node;
1023
1024 Exit:
1025   if(NamespaceData != NULL) {
1026     FreePool (NamespaceData);
1027   }
1028
1029   if (EFI_ERROR (Status)) {
1030     FreePool (Node);
1031   }
1032
1033   return Status;
1034 }
1035