MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressPassthru.c

   1 /** @file
   2   NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows
   3   NVM Express specification.
   4
   5   (C) Copyright 2014 Hewlett-Packard Development Company, L.P.<BR>
   6   Copyright (c) 2013 - 2015, Intel Corporation. All rights reserved.<BR>
   7   This program and the accompanying materials
   8   are licensed and made available under the terms and conditions of the BSD License
   9   which accompanies this distribution.  The full text of the license may be found at
  10   http://opensource.org/licenses/bsd-license.php.
  11
  12   THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  13   WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  14
  15 **/
  16
  17 #include "NvmExpress.h"
  18
  19 //
  20 // Page size should be set in the Controller Configuration register
  21 // during controller init, and the controller configuration save in
  22 // the controller's private data.  The Max and Min supported page sizes
  23 // for the controller are specified in the Controller Capabilities register.
  24 //
  25
  26 GLOBAL_REMOVE_IF_UNREFERENCED NVM_EXPRESS_PASS_THRU_MODE gNvmExpressPassThruMode = {
  27   0,
  28   NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL | NVM_EXPRESS_PASS_THRU_ATTRIBUTES_CMD_SET_NVME,
  29   sizeof (UINTN),
  30   0x10000,
  31   0,
  32   0
  33 };
  34
  35
  36 /**
  37   Dump the execution status from a given completion queue entry.
  38
  39   @param[in]     Cq               A pointer to the NVME_CQ item.
  40
  41 **/
  42 VOID
  43 NvmeDumpStatus (
  44   IN NVME_CQ             *Cq
  45   )
  46 {
  47   DEBUG ((EFI_D_VERBOSE, "Dump NVMe Completion Entry Status from [0x%x]:\n", Cq));
  48
  49   DEBUG ((EFI_D_VERBOSE, "  SQ Identifier : [0x%x], Phase Tag : [%d], Cmd Identifier : [0x%x]\n", Cq->Sqid, Cq->Pt, Cq->Cid));
  50
  51   DEBUG ((EFI_D_VERBOSE, "  NVMe Cmd Execution Result - "));
  52
  53   switch (Cq->Sct) {
  54     case 0x0:
  55       switch (Cq->Sc) {
  56         case 0x0:
  57           DEBUG ((EFI_D_VERBOSE, "Successful Completion\n"));
  58           break;
  59         case 0x1:
  60           DEBUG ((EFI_D_VERBOSE, "Invalid Command Opcode\n"));
  61           break;
  62         case 0x2:
  63           DEBUG ((EFI_D_VERBOSE, "Invalid Field in Command\n"));
  64           break;
  65         case 0x3:
  66           DEBUG ((EFI_D_VERBOSE, "Command ID Conflict\n"));
  67           break;
  68         case 0x4:
  69           DEBUG ((EFI_D_VERBOSE, "Data Transfer Error\n"));
  70           break;
  71         case 0x5:
  72           DEBUG ((EFI_D_VERBOSE, "Commands Aborted due to Power Loss Notification\n"));
  73           break;
  74         case 0x6:
  75           DEBUG ((EFI_D_VERBOSE, "Internal Device Error\n"));
  76           break;
  77         case 0x7:
  78           DEBUG ((EFI_D_VERBOSE, "Command Abort Requested\n"));
  79           break;
  80         case 0x8:
  81           DEBUG ((EFI_D_VERBOSE, "Command Aborted due to SQ Deletion\n"));
  82           break;
  83         case 0x9:
  84           DEBUG ((EFI_D_VERBOSE, "Command Aborted due to Failed Fused Command\n"));
  85           break;
  86         case 0xA:
  87           DEBUG ((EFI_D_VERBOSE, "Command Aborted due to Missing Fused Command\n"));
  88           break;
  89         case 0xB:
  90           DEBUG ((EFI_D_VERBOSE, "Invalid Namespace or Format\n"));
  91           break;
  92         case 0xC:
  93           DEBUG ((EFI_D_VERBOSE, "Command Sequence Error\n"));
  94           break;
  95         case 0xD:
  96           DEBUG ((EFI_D_VERBOSE, "Invalid SGL Last Segment Descriptor\n"));
  97           break;
  98         case 0xE:
  99           DEBUG ((EFI_D_VERBOSE, "Invalid Number of SGL Descriptors\n"));
 100           break;
 101         case 0xF:
 102           DEBUG ((EFI_D_VERBOSE, "Data SGL Length Invalid\n"));
 103           break;
 104         case 0x10:
 105           DEBUG ((EFI_D_VERBOSE, "Metadata SGL Length Invalid\n"));
 106           break;
 107         case 0x11:
 108           DEBUG ((EFI_D_VERBOSE, "SGL Descriptor Type Invalid\n"));
 109           break;
 110         case 0x80:
 111           DEBUG ((EFI_D_VERBOSE, "LBA Out of Range\n"));
 112           break;
 113         case 0x81:
 114           DEBUG ((EFI_D_VERBOSE, "Capacity Exceeded\n"));
 115           break;
 116         case 0x82:
 117           DEBUG ((EFI_D_VERBOSE, "Namespace Not Ready\n"));
 118           break;
 119         case 0x83:
 120           DEBUG ((EFI_D_VERBOSE, "Reservation Conflict\n"));
 121           break;
 122       }
 123       break;
 124
 125     case 0x1:
 126       switch (Cq->Sc) {
 127         case 0x0:
 128           DEBUG ((EFI_D_VERBOSE, "Completion Queue Invalid\n"));
 129           break;
 130         case 0x1:
 131           DEBUG ((EFI_D_VERBOSE, "Invalid Queue Identifier\n"));
 132           break;
 133         case 0x2:
 134           DEBUG ((EFI_D_VERBOSE, "Maximum Queue Size Exceeded\n"));
 135           break;
 136         case 0x3:
 137           DEBUG ((EFI_D_VERBOSE, "Abort Command Limit Exceeded\n"));
 138           break;
 139         case 0x5:
 140           DEBUG ((EFI_D_VERBOSE, "Asynchronous Event Request Limit Exceeded\n"));
 141           break;
 142         case 0x6:
 143           DEBUG ((EFI_D_VERBOSE, "Invalid Firmware Slot\n"));
 144           break;
 145         case 0x7:
 146           DEBUG ((EFI_D_VERBOSE, "Invalid Firmware Image\n"));
 147           break;
 148         case 0x8:
 149           DEBUG ((EFI_D_VERBOSE, "Invalid Interrupt Vector\n"));
 150           break;
 151         case 0x9:
 152           DEBUG ((EFI_D_VERBOSE, "Invalid Log Page\n"));
 153           break;
 154         case 0xA:
 155           DEBUG ((EFI_D_VERBOSE, "Invalid Format\n"));
 156           break;
 157         case 0xB:
 158           DEBUG ((EFI_D_VERBOSE, "Firmware Application Requires Conventional Reset\n"));
 159           break;
 160         case 0xC:
 161           DEBUG ((EFI_D_VERBOSE, "Invalid Queue Deletion\n"));
 162           break;
 163         case 0xD:
 164           DEBUG ((EFI_D_VERBOSE, "Feature Identifier Not Saveable\n"));
 165           break;
 166         case 0xE:
 167           DEBUG ((EFI_D_VERBOSE, "Feature Not Changeable\n"));
 168           break;
 169         case 0xF:
 170           DEBUG ((EFI_D_VERBOSE, "Feature Not Namespace Specific\n"));
 171           break;
 172         case 0x10:
 173           DEBUG ((EFI_D_VERBOSE, "Firmware Application Requires NVM Subsystem Reset\n"));
 174           break;
 175         case 0x80:
 176           DEBUG ((EFI_D_VERBOSE, "Conflicting Attributes\n"));
 177           break;
 178         case 0x81:
 179           DEBUG ((EFI_D_VERBOSE, "Invalid Protection Information\n"));
 180           break;
 181         case 0x82:
 182           DEBUG ((EFI_D_VERBOSE, "Attempted Write to Read Only Range\n"));
 183           break;
 184       }
 185       break;
 186
 187     case 0x2:
 188       switch (Cq->Sc) {
 189         case 0x80:
 190           DEBUG ((EFI_D_VERBOSE, "Write Fault\n"));
 191           break;
 192         case 0x81:
 193           DEBUG ((EFI_D_VERBOSE, "Unrecovered Read Error\n"));
 194           break;
 195         case 0x82:
 196           DEBUG ((EFI_D_VERBOSE, "End-to-end Guard Check Error\n"));
 197           break;
 198         case 0x83:
 199           DEBUG ((EFI_D_VERBOSE, "End-to-end Application Tag Check Error\n"));
 200           break;
 201         case 0x84:
 202           DEBUG ((EFI_D_VERBOSE, "End-to-end Reference Tag Check Error\n"));
 203           break;
 204         case 0x85:
 205           DEBUG ((EFI_D_VERBOSE, "Compare Failure\n"));
 206           break;
 207         case 0x86:
 208           DEBUG ((EFI_D_VERBOSE, "Access Denied\n"));
 209           break;
 210       }
 211       break;
 212
 213     default:
 214       break;
 215   }
 216 }
 217
 218 /**
 219   Create PRP lists for data transfer which is larger than 2 memory pages.
 220   Note here we calcuate the number of required PRP lists and allocate them at one time.
 221
 222   @param[in]     PciIo               A pointer to the EFI_PCI_IO_PROTOCOL instance.
 223   @param[in]     PhysicalAddr        The physical base address of data buffer.
 224   @param[in]     Pages               The number of pages to be transfered.
 225   @param[out]    PrpListHost         The host base address of PRP lists.
 226   @param[in,out] PrpListNo           The number of PRP List.
 227   @param[out]    Mapping             The mapping value returned from PciIo.Map().
 228
 229   @retval The pointer to the first PRP List of the PRP lists.
 230
 231 **/
 232 VOID*
 233 NvmeCreatePrpList (
 234   IN     EFI_PCI_IO_PROTOCOL          *PciIo,
 235   IN     EFI_PHYSICAL_ADDRESS         PhysicalAddr,
 236   IN     UINTN                        Pages,
 237      OUT VOID                         **PrpListHost,
 238   IN OUT UINTN                        *PrpListNo,
 239      OUT VOID                         **Mapping
 240   )
 241 {
 242   UINTN                       PrpEntryNo;
 243   UINT64                      PrpListBase;
 244   UINTN                       PrpListIndex;
 245   UINTN                       PrpEntryIndex;
 246   UINT64                      Remainder;
 247   EFI_PHYSICAL_ADDRESS        PrpListPhyAddr;
 248   UINTN                       Bytes;
 249   EFI_STATUS                  Status;
 250
 251   //
 252   // The number of Prp Entry in a memory page.
 253   //
 254   PrpEntryNo = EFI_PAGE_SIZE / sizeof (UINT64);
 255
 256   //
 257   // Calculate total PrpList number.
 258   //
 259   *PrpListNo = (UINTN)DivU64x64Remainder ((UINT64)Pages, (UINT64)PrpEntryNo - 1, &Remainder);
 260   if (*PrpListNo == 0) {
 261     *PrpListNo = 1;
 262   } else if ((Remainder != 0) && (Remainder != 1)) {
 263     *PrpListNo += 1;
 264   } else if (Remainder == 1) {
 265     Remainder = PrpEntryNo;
 266   } else if (Remainder == 0) {
 267     Remainder = PrpEntryNo - 1;
 268   }
 269
 270   Status = PciIo->AllocateBuffer (
 271                     PciIo,
 272                     AllocateAnyPages,
 273                     EfiBootServicesData,
 274                     *PrpListNo,
 275                     PrpListHost,
 276                     0
 277                     );
 278
 279   if (EFI_ERROR (Status)) {
 280     return NULL;
 281   }
 282
 283   Bytes = EFI_PAGES_TO_SIZE (*PrpListNo);
 284   Status = PciIo->Map (
 285                     PciIo,
 286                     EfiPciIoOperationBusMasterCommonBuffer,
 287                     *PrpListHost,
 288                     &Bytes,
 289                     &PrpListPhyAddr,
 290                     Mapping
 291                     );
 292
 293   if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (*PrpListNo))) {
 294     DEBUG ((EFI_D_ERROR, "NvmeCreatePrpList: create PrpList failure!\n"));
 295     goto EXIT;
 296   }
 297   //
 298   // Fill all PRP lists except of last one.
 299   //
 300   ZeroMem (*PrpListHost, Bytes);
 301   for (PrpListIndex = 0; PrpListIndex < *PrpListNo - 1; ++PrpListIndex) {
 302     PrpListBase = *(UINT64*)PrpListHost + PrpListIndex * EFI_PAGE_SIZE;
 303
 304     for (PrpEntryIndex = 0; PrpEntryIndex < PrpEntryNo; ++PrpEntryIndex) {
 305       if (PrpEntryIndex != PrpEntryNo - 1) {
 306         //
 307         // Fill all PRP entries except of last one.
 308         //
 309         *((UINT64*)(UINTN)PrpListBase + PrpEntryIndex) = PhysicalAddr;
 310         PhysicalAddr += EFI_PAGE_SIZE;
 311       } else {
 312         //
 313         // Fill last PRP entries with next PRP List pointer.
 314         //
 315         *((UINT64*)(UINTN)PrpListBase + PrpEntryIndex) = PrpListPhyAddr + (PrpListIndex + 1) * EFI_PAGE_SIZE;
 316       }
 317     }
 318   }
 319   //
 320   // Fill last PRP list.
 321   //
 322   PrpListBase = *(UINT64*)PrpListHost + PrpListIndex * EFI_PAGE_SIZE;
 323   for (PrpEntryIndex = 0; PrpEntryIndex < Remainder; ++PrpEntryIndex) {
 324     *((UINT64*)(UINTN)PrpListBase + PrpEntryIndex) = PhysicalAddr;
 325     PhysicalAddr += EFI_PAGE_SIZE;
 326   }
 327
 328   return (VOID*)(UINTN)PrpListPhyAddr;
 329
 330 EXIT:
 331   PciIo->FreeBuffer (PciIo, *PrpListNo, *PrpListHost);
 332   return NULL;
 333 }
 334
 335
 336 /**
 337   Sends an NVM Express Command Packet to an NVM Express controller or namespace. This function supports
 338   both blocking I/O and nonblocking I/O. The blocking I/O functionality is required, and the nonblocking
 339   I/O functionality is optional.
 340
 341   @param[in]     This                A pointer to the NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
 342   @param[in]     NamespaceId         Is a 32 bit Namespace ID to which the Express HCI command packet will be sent.
 343                                      A value of 0 denotes the NVM Express controller, a value of all 0FFh in the namespace
 344                                      ID specifies that the command packet should be sent to all valid namespaces.
 345   @param[in]     NamespaceUuid       Is a 64 bit Namespace UUID to which the Express HCI command packet will be sent.
 346                                      A value of 0 denotes the NVM Express controller, a value of all 0FFh in the namespace
 347                                      UUID specifies that the command packet should be sent to all valid namespaces.
 348   @param[in,out] Packet              A pointer to the NVM Express HCI Command Packet to send to the NVMe namespace specified
 349                                      by NamespaceId.
 350   @param[in]     Event               If nonblocking I/O is not supported then Event is ignored, and blocking I/O is performed.
 351                                      If Event is NULL, then blocking I/O is performed. If Event is not NULL and non blocking I/O
 352                                      is supported, then nonblocking I/O is performed, and Event will be signaled when the NVM
 353                                      Express Command Packet completes.
 354
 355   @retval EFI_SUCCESS                The NVM Express Command Packet was sent by the host. TransferLength bytes were transferred
 356                                      to, or from DataBuffer.
 357   @retval EFI_BAD_BUFFER_SIZE        The NVM Express Command Packet was not executed. The number of bytes that could be transferred
 358                                      is returned in TransferLength.
 359   @retval EFI_NOT_READY              The NVM Express Command Packet could not be sent because the controller is not ready. The caller
 360                                      may retry again later.
 361   @retval EFI_DEVICE_ERROR           A device error occurred while attempting to send the NVM Express Command Packet.
 362   @retval EFI_INVALID_PARAMETER      Namespace, or the contents of NVM_EXPRESS_PASS_THRU_COMMAND_PACKET are invalid. The NVM
 363                                      Express Command Packet was not sent, so no additional status information is available.
 364   @retval EFI_UNSUPPORTED            The command described by the NVM Express Command Packet is not supported by the host adapter.
 365                                      The NVM Express Command Packet was not sent, so no additional status information is available.
 366   @retval EFI_TIMEOUT                A timeout occurred while waiting for the NVM Express Command Packet to execute.
 367
 368 **/
 369 EFI_STATUS
 370 EFIAPI
 371 NvmExpressPassThru (
 372   IN     NVM_EXPRESS_PASS_THRU_PROTOCOL              *This,
 373   IN     UINT32                                      NamespaceId,
 374   IN     UINT64                                      NamespaceUuid,
 375   IN OUT NVM_EXPRESS_PASS_THRU_COMMAND_PACKET        *Packet,
 376   IN     EFI_EVENT                                   Event OPTIONAL
 377   )
 378 {
 379   NVME_CONTROLLER_PRIVATE_DATA  *Private;
 380   EFI_STATUS                    Status;
 381   EFI_PCI_IO_PROTOCOL           *PciIo;
 382   NVME_SQ                       *Sq;
 383   NVME_CQ                       *Cq;
 384   UINT8                         Qid;
 385   UINT32                        Bytes;
 386   UINT16                        Offset;
 387   EFI_EVENT                     TimerEvent;
 388   EFI_PCI_IO_PROTOCOL_OPERATION Flag;
 389   EFI_PHYSICAL_ADDRESS          PhyAddr;
 390   VOID                          *MapData;
 391   VOID                          *MapMeta;
 392   VOID                          *MapPrpList;
 393   UINTN                         MapLength;
 394   UINT64                        *Prp;
 395   VOID                          *PrpListHost;
 396   UINTN                         PrpListNo;
 397   UINT32                        Data;
 398
 399   //
 400   // check the data fields in Packet parameter.
 401   //
 402   if ((This == NULL) || (Packet == NULL)) {
 403     return EFI_INVALID_PARAMETER;
 404   }
 405
 406   if ((Packet->NvmeCmd == NULL) || (Packet->NvmeResponse == NULL)) {
 407     return EFI_INVALID_PARAMETER;
 408   }
 409
 410   if (Packet->QueueId != NVME_ADMIN_QUEUE && Packet->QueueId != NVME_IO_QUEUE) {
 411     return EFI_INVALID_PARAMETER;
 412   }
 413
 414   Private     = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
 415   PciIo       = Private->PciIo;
 416   MapData     = NULL;
 417   MapMeta     = NULL;
 418   MapPrpList  = NULL;
 419   PrpListHost = NULL;
 420   PrpListNo   = 0;
 421   Prp         = NULL;
 422   TimerEvent  = NULL;
 423   Status      = EFI_SUCCESS;
 424
 425   Qid = Packet->QueueId;
 426   Sq  = Private->SqBuffer[Qid] + Private->SqTdbl[Qid].Sqt;
 427   Cq  = Private->CqBuffer[Qid] + Private->CqHdbl[Qid].Cqh;
 428
 429   if (Packet->NvmeCmd->Nsid != NamespaceId) {
 430     return EFI_INVALID_PARAMETER;
 431   }
 432
 433   ZeroMem (Sq, sizeof (NVME_SQ));
 434   Sq->Opc  = Packet->NvmeCmd->Cdw0.Opcode;
 435   Sq->Fuse = Packet->NvmeCmd->Cdw0.FusedOperation;
 436   Sq->Cid  = Packet->NvmeCmd->Cdw0.Cid;
 437   Sq->Nsid = Packet->NvmeCmd->Nsid;
 438
 439   //
 440   // Currently we only support PRP for data transfer, SGL is NOT supported.
 441   //
 442   ASSERT (Sq->Psdt == 0);
 443   if (Sq->Psdt != 0) {
 444     DEBUG ((EFI_D_ERROR, "NvmExpressPassThru: doesn't support SGL mechanism\n"));
 445     return EFI_UNSUPPORTED;
 446   }
 447
 448   Sq->Prp[0] = (UINT64)(UINTN)Packet->TransferBuffer;
 449   //
 450   // If the NVMe cmd has data in or out, then mapping the user buffer to the PCI controller specific addresses.
 451   // Note here we don't handle data buffer for CreateIOSubmitionQueue and CreateIOCompletionQueue cmds because
 452   // these two cmds are special which requires their data buffer must support simultaneous access by both the
 453   // processor and a PCI Bus Master. It's caller's responsbility to ensure this.
 454   //
 455   if (((Sq->Opc & (BIT0 | BIT1)) != 0) && (Sq->Opc != NVME_ADMIN_CRIOCQ_OPC) && (Sq->Opc != NVME_ADMIN_CRIOSQ_OPC)) {
 456     if ((Sq->Opc & BIT0) != 0) {
 457       Flag = EfiPciIoOperationBusMasterRead;
 458     } else {
 459       Flag = EfiPciIoOperationBusMasterWrite;
 460     }
 461
 462     MapLength = Packet->TransferLength;
 463     Status = PciIo->Map (
 464                       PciIo,
 465                       Flag,
 466                       Packet->TransferBuffer,
 467                       &MapLength,
 468                       &PhyAddr,
 469                       &MapData
 470                       );
 471     if (EFI_ERROR (Status) || (Packet->TransferLength != MapLength)) {
 472       return EFI_OUT_OF_RESOURCES;
 473     }
 474
 475     Sq->Prp[0] = PhyAddr;
 476     Sq->Prp[1] = 0;
 477
 478     MapLength = Packet->MetadataLength;
 479     if(Packet->MetadataBuffer != NULL) {
 480       MapLength = Packet->MetadataLength;
 481       Status = PciIo->Map (
 482                         PciIo,
 483                         Flag,
 484                         Packet->MetadataBuffer,
 485                         &MapLength,
 486                         &PhyAddr,
 487                         &MapMeta
 488                         );
 489       if (EFI_ERROR (Status) || (Packet->MetadataLength != MapLength)) {
 490         PciIo->Unmap (
 491                  PciIo,
 492                  MapData
 493                  );
 494
 495         return EFI_OUT_OF_RESOURCES;
 496       }
 497       Sq->Mptr = PhyAddr;
 498     }
 499   }
 500   //
 501   // If the buffer size spans more than two memory pages (page size as defined in CC.Mps),
 502   // then build a PRP list in the second PRP submission queue entry.
 503   //
 504   Offset = ((UINT16)Sq->Prp[0]) & (EFI_PAGE_SIZE - 1);
 505   Bytes  = Packet->TransferLength;
 506
 507   if ((Offset + Bytes) > (EFI_PAGE_SIZE * 2)) {
 508     //
 509     // Create PrpList for remaining data buffer.
 510     //
 511     PhyAddr = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
 512     Prp = NvmeCreatePrpList (PciIo, PhyAddr, EFI_SIZE_TO_PAGES(Offset + Bytes) - 1, &PrpListHost, &PrpListNo, &MapPrpList);
 513     if (Prp == NULL) {
 514       goto EXIT;
 515     }
 516
 517     Sq->Prp[1] = (UINT64)(UINTN)Prp;
 518   } else if ((Offset + Bytes) > EFI_PAGE_SIZE) {
 519     Sq->Prp[1] = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
 520   }
 521
 522   if(Packet->NvmeCmd->Flags & CDW10_VALID) {
 523     Sq->Payload.Raw.Cdw10 = Packet->NvmeCmd->Cdw10;
 524   }
 525   if(Packet->NvmeCmd->Flags & CDW11_VALID) {
 526     Sq->Payload.Raw.Cdw11 = Packet->NvmeCmd->Cdw11;
 527   }
 528   if(Packet->NvmeCmd->Flags & CDW12_VALID) {
 529     Sq->Payload.Raw.Cdw12 = Packet->NvmeCmd->Cdw12;
 530   }
 531   if(Packet->NvmeCmd->Flags & CDW13_VALID) {
 532     Sq->Payload.Raw.Cdw13 = Packet->NvmeCmd->Cdw13;
 533   }
 534   if(Packet->NvmeCmd->Flags & CDW14_VALID) {
 535     Sq->Payload.Raw.Cdw14 = Packet->NvmeCmd->Cdw14;
 536   }
 537   if(Packet->NvmeCmd->Flags & CDW15_VALID) {
 538     Sq->Payload.Raw.Cdw15 = Packet->NvmeCmd->Cdw15;
 539   }
 540
 541   //
 542   // Ring the submission queue doorbell.
 543   //
 544   Private->SqTdbl[Qid].Sqt ^= 1;
 545   Data = ReadUnaligned32 ((UINT32*)&Private->SqTdbl[Qid]);
 546   PciIo->Mem.Write (
 547                PciIo,
 548                EfiPciIoWidthUint32,
 549                NVME_BAR,
 550                NVME_SQTDBL_OFFSET(Qid, Private->Cap.Dstrd),
 551                1,
 552                &Data
 553                );
 554
 555   Status = gBS->CreateEvent (
 556                   EVT_TIMER,
 557                   TPL_CALLBACK,
 558                   NULL,
 559                   NULL,
 560                   &TimerEvent
 561                   );
 562   if (EFI_ERROR (Status)) {
 563     goto EXIT;
 564   }
 565
 566   Status = gBS->SetTimer(TimerEvent, TimerRelative, Packet->CommandTimeout);
 567
 568   if (EFI_ERROR(Status)) {
 569     Packet->ControllerStatus = NVM_EXPRESS_STATUS_CONTROLLER_DEVICE_ERROR;
 570     goto EXIT;
 571   }
 572
 573   //
 574   // Wait for completion queue to get filled in.
 575   //
 576   Status = EFI_TIMEOUT;
 577   Packet->ControllerStatus = NVM_EXPRESS_STATUS_CONTROLLER_TIMEOUT_COMMAND;
 578   while (EFI_ERROR (gBS->CheckEvent (TimerEvent))) {
 579     if (Cq->Pt != Private->Pt[Qid]) {
 580       Status = EFI_SUCCESS;
 581       Packet->ControllerStatus = NVM_EXPRESS_STATUS_CONTROLLER_READY;
 582       break;
 583     }
 584   }
 585
 586   if ((Private->CqHdbl[Qid].Cqh ^= 1) == 0) {
 587     Private->Pt[Qid] ^= 1;
 588   }
 589
 590   //
 591   // Copy the Respose Queue entry for this command to the callers response buffer
 592   //
 593   CopyMem(Packet->NvmeResponse, Cq, sizeof(NVM_EXPRESS_RESPONSE));
 594
 595   //
 596   // Dump every completion entry status for debugging.
 597   //
 598   DEBUG_CODE_BEGIN();
 599     NvmeDumpStatus(Cq);
 600   DEBUG_CODE_END();
 601
 602   Data = ReadUnaligned32 ((UINT32*)&Private->CqHdbl[Qid]);
 603   PciIo->Mem.Write (
 604                PciIo,
 605                EfiPciIoWidthUint32,
 606                NVME_BAR,
 607                NVME_CQHDBL_OFFSET(Qid, Private->Cap.Dstrd),
 608                1,
 609                &Data
 610                );
 611
 612 EXIT:
 613   if (MapData != NULL) {
 614     PciIo->Unmap (
 615              PciIo,
 616              MapData
 617              );
 618   }
 619
 620   if (MapMeta != NULL) {
 621     PciIo->Unmap (
 622              PciIo,
 623              MapMeta
 624              );
 625   }
 626
 627   if (MapPrpList != NULL) {
 628     PciIo->Unmap (
 629              PciIo,
 630              MapPrpList
 631              );
 632   }
 633
 634   if (Prp != NULL) {
 635     PciIo->FreeBuffer (PciIo, PrpListNo, PrpListHost);
 636   }
 637
 638   if (TimerEvent != NULL) {
 639     gBS->CloseEvent (TimerEvent);
 640   }
 641   return Status;
 642 }
 643
 644 /**
 645   Used to retrieve the list of namespaces defined on an NVM Express controller.
 646
 647   The NVM_EXPRESS_PASS_THRU_PROTOCOL.GetNextNamespace() function retrieves a list of namespaces
 648   defined on an NVM Express controller. If on input a NamespaceID is specified by all 0xFF in the
 649   namespace buffer, then the first namespace defined on the NVM Express controller is returned in
 650   NamespaceID, and a status of EFI_SUCCESS is returned.
 651
 652   If NamespaceId is a Namespace value that was returned on a previous call to GetNextNamespace(),
 653   then the next valid NamespaceId  for an NVM Express SSD namespace on the NVM Express controller
 654   is returned in NamespaceId, and EFI_SUCCESS is returned.
 655
 656   If Namespace array is not a 0xFFFFFFFF and NamespaceId was not returned on a previous call to
 657   GetNextNamespace(), then EFI_INVALID_PARAMETER is returned.
 658
 659   If NamespaceId is the NamespaceId of the last SSD namespace on the NVM Express controller, then
 660   EFI_NOT_FOUND is returned
 661
 662   @param[in]     This           A pointer to the NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
 663   @param[in,out] NamespaceId    On input, a pointer to a legal NamespaceId for an NVM Express
 664                                 namespace present on the NVM Express controller. On output, a
 665                                 pointer to the next NamespaceId of an NVM Express namespace on
 666                                 an NVM Express controller. An input value of 0xFFFFFFFF retrieves
 667                                 the first NamespaceId for an NVM Express namespace present on an
 668                                 NVM Express controller.
 669   @param[out]    NamespaceUuid  On output, the UUID associated with the next namespace, if a UUID
 670                                 is defined for that NamespaceId, otherwise, zero is returned in
 671                                 this parameter. If the caller does not require a UUID, then a NULL
 672                                 pointer may be passed.
 673
 674   @retval EFI_SUCCESS           The NamespaceId of the next Namespace was returned.
 675   @retval EFI_NOT_FOUND         There are no more namespaces defined on this controller.
 676   @retval EFI_INVALID_PARAMETER Namespace array is not a 0xFFFFFFFF and NamespaceId was not returned
 677                                 on a previous call to GetNextNamespace().
 678
 679 **/
 680 EFI_STATUS
 681 EFIAPI
 682 NvmExpressGetNextNamespace (
 683   IN     NVM_EXPRESS_PASS_THRU_PROTOCOL              *This,
 684   IN OUT UINT32                                      *NamespaceId,
 685      OUT UINT64                                      *NamespaceUuid  OPTIONAL
 686   )
 687 {
 688   NVME_CONTROLLER_PRIVATE_DATA     *Private;
 689   NVME_ADMIN_NAMESPACE_DATA        *NamespaceData;
 690   UINT32                           NextNamespaceId;
 691   EFI_STATUS                       Status;
 692
 693   if ((This == NULL) || (NamespaceId == NULL)) {
 694     return EFI_INVALID_PARAMETER;
 695   }
 696
 697   NamespaceData = NULL;
 698   Status        = EFI_NOT_FOUND;
 699
 700   Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
 701   //
 702   // If the NamespaceId input value is 0xFFFFFFFF, then get the first valid namespace ID
 703   //
 704   if (*NamespaceId == 0xFFFFFFFF) {
 705     //
 706     // Start with the first namespace ID
 707     //
 708     NextNamespaceId = 1;
 709     //
 710     // Allocate buffer for Identify Namespace data.
 711     //
 712     NamespaceData = (NVME_ADMIN_NAMESPACE_DATA *)AllocateZeroPool (sizeof (NVME_ADMIN_NAMESPACE_DATA));
 713
 714     if (NamespaceData == NULL) {
 715       return EFI_NOT_FOUND;
 716     }
 717
 718     Status = NvmeIdentifyNamespace (Private, NextNamespaceId, NamespaceData);
 719     if (EFI_ERROR(Status)) {
 720       goto Done;
 721     }
 722
 723     *NamespaceId = NextNamespaceId;
 724     if (NamespaceUuid != NULL) {
 725       *NamespaceUuid = NamespaceData->Eui64;
 726     }
 727   } else {
 728     if (*NamespaceId >= Private->ControllerData->Nn) {
 729       return EFI_INVALID_PARAMETER;
 730     }
 731
 732     NextNamespaceId = *NamespaceId + 1;
 733     //
 734     // Allocate buffer for Identify Namespace data.
 735     //
 736     NamespaceData = (NVME_ADMIN_NAMESPACE_DATA *)AllocateZeroPool (sizeof (NVME_ADMIN_NAMESPACE_DATA));
 737     if (NamespaceData == NULL) {
 738       return EFI_NOT_FOUND;
 739     }
 740
 741     Status = NvmeIdentifyNamespace (Private, NextNamespaceId, NamespaceData);
 742     if (EFI_ERROR(Status)) {
 743       goto Done;
 744     }
 745
 746     *NamespaceId = NextNamespaceId;
 747     if (NamespaceUuid != NULL) {
 748       *NamespaceUuid = NamespaceData->Eui64;
 749     }
 750   }
 751
 752 Done:
 753   if (NamespaceData != NULL) {
 754     FreePool(NamespaceData);
 755   }
 756
 757   return Status;
 758 }
 759
 760 /**
 761   Used to translate a device path node to a Namespace ID and Namespace UUID.
 762
 763   The NVM_EXPRESS_PASS_THRU_PROTOCOL.GetNamwspace() function determines the Namespace ID and Namespace UUID
 764   associated with the NVM Express SSD namespace described by DevicePath. If DevicePath is a device path node type
 765   that the NVM Express Pass Thru driver supports, then the NVM Express Pass Thru driver will attempt to translate
 766   the contents DevicePath into a Namespace ID and UUID. If this translation is successful, then that Namespace ID
 767   and UUID are returned in NamespaceID and NamespaceUUID, and EFI_SUCCESS is returned.
 768
 769   @param[in]  This                A pointer to the NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
 770   @param[in]  DevicePath          A pointer to the device path node that describes an NVM Express namespace on
 771                                   the NVM Express controller.
 772   @param[out] NamespaceId         The NVM Express namespace ID contained in the device path node.
 773   @param[out] NamespaceUuid       The NVM Express namespace contained in the device path node.
 774
 775   @retval EFI_SUCCESS             DevicePath was successfully translated to NamespaceId and NamespaceUuid.
 776   @retval EFI_INVALID_PARAMETER   If DevicePath, NamespaceId, or NamespaceUuid are NULL, then EFI_INVALID_PARAMETER
 777                                   is returned.
 778   @retval EFI_UNSUPPORTED         If DevicePath is not a device path node type that the NVM Express Pass Thru driver
 779                                   supports, then EFI_UNSUPPORTED is returned.
 780   @retval EFI_NOT_FOUND           If DevicePath is a device path node type that the Nvm Express Pass Thru driver
 781                                   supports, but there is not a valid translation from DevicePath to a NamespaceID
 782                                   and NamespaceUuid, then EFI_NOT_FOUND is returned.
 783 **/
 784 EFI_STATUS
 785 EFIAPI
 786 NvmExpressGetNamespace (
 787   IN     NVM_EXPRESS_PASS_THRU_PROTOCOL              *This,
 788   IN     EFI_DEVICE_PATH_PROTOCOL                    *DevicePath,
 789      OUT UINT32                                      *NamespaceId,
 790      OUT UINT64                                      *NamespaceUuid
 791   )
 792 {
 793   NVME_NAMESPACE_DEVICE_PATH       *Node;
 794
 795   if ((This == NULL) || (DevicePath == NULL) || (NamespaceId == NULL) || (NamespaceUuid == NULL)) {
 796     return EFI_INVALID_PARAMETER;
 797   }
 798
 799   if (DevicePath->Type != MESSAGING_DEVICE_PATH) {
 800     return EFI_UNSUPPORTED;
 801   }
 802
 803   Node = (NVME_NAMESPACE_DEVICE_PATH *)DevicePath;
 804
 805   if (DevicePath->SubType == MSG_NVME_NAMESPACE_DP) {
 806     if (DevicePathNodeLength(DevicePath) != sizeof(NVME_NAMESPACE_DEVICE_PATH)) {
 807       return EFI_NOT_FOUND;
 808     }
 809
 810     *NamespaceId   = Node->NamespaceId;
 811     *NamespaceUuid = Node->NamespaceUuid;
 812
 813     return EFI_SUCCESS;
 814   } else {
 815     return EFI_UNSUPPORTED;
 816   }
 817 }
 818
 819 /**
 820   Used to allocate and build a device path node for an NVM Express namespace on an NVM Express controller.
 821
 822   The NVM_EXPRESS_PASS_THRU_PROTOCOL.BuildDevicePath() function allocates and builds a single device
 823   path node for the NVM Express namespace specified by NamespaceId.
 824
 825   If the namespace device specified by NamespaceId is not valid , then EFI_NOT_FOUND is returned.
 826
 827   If DevicePath is NULL, then EFI_INVALID_PARAMETER is returned.
 828
 829   If there are not enough resources to allocate the device path node, then EFI_OUT_OF_RESOURCES is returned.
 830
 831   Otherwise, DevicePath is allocated with the boot service AllocatePool(), the contents of DevicePath are
 832   initialized to describe the NVM Express namespace specified by NamespaceId, and EFI_SUCCESS is returned.
 833
 834   @param[in]     This                A pointer to the NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
 835   @param[in]     NamespaceId         The NVM Express namespace ID  for which a device path node is to be
 836                                      allocated and built. Caller must set the NamespaceId to zero if the
 837                                      device path node will contain a valid UUID.
 838   @param[in]     NamespaceUuid       The NVM Express namespace UUID for which a device path node is to be
 839                                      allocated and built. UUID will only be valid of the Namespace ID is zero.
 840   @param[in,out] DevicePath          A pointer to a single device path node that describes the NVM Express
 841                                      namespace specified by NamespaceId. This function is responsible for
 842                                      allocating the buffer DevicePath with the boot service AllocatePool().
 843                                      It is the caller's responsibility to free DevicePath when the caller
 844                                      is finished with DevicePath.
 845   @retval EFI_SUCCESS                The device path node that describes the NVM Express namespace specified
 846                                      by NamespaceId was allocated and returned in DevicePath.
 847   @retval EFI_NOT_FOUND              The NVM Express namespace specified by NamespaceId does not exist on the
 848                                      NVM Express controller.
 849   @retval EFI_INVALID_PARAMETER      DevicePath is NULL.
 850   @retval EFI_OUT_OF_RESOURCES       There are not enough resources to allocate the DevicePath node.
 851
 852 **/
 853 EFI_STATUS
 854 EFIAPI
 855 NvmExpressBuildDevicePath (
 856   IN     NVM_EXPRESS_PASS_THRU_PROTOCOL              *This,
 857   IN     UINT32                                      NamespaceId,
 858   IN     UINT64                                      NamespaceUuid,
 859   IN OUT EFI_DEVICE_PATH_PROTOCOL                    **DevicePath
 860   )
 861 {
 862   NVME_NAMESPACE_DEVICE_PATH     *Node;
 863
 864   //
 865   // Validate parameters
 866   //
 867   if ((This == NULL) || (DevicePath == NULL)) {
 868     return EFI_INVALID_PARAMETER;
 869   }
 870
 871   if (NamespaceId == 0) {
 872     return EFI_NOT_FOUND;
 873   }
 874
 875   Node = (NVME_NAMESPACE_DEVICE_PATH *)AllocateZeroPool (sizeof (NVME_NAMESPACE_DEVICE_PATH));
 876
 877   if (Node == NULL) {
 878     return EFI_OUT_OF_RESOURCES;
 879   }
 880
 881   Node->Header.Type    = MESSAGING_DEVICE_PATH;
 882   Node->Header.SubType = MSG_NVME_NAMESPACE_DP;
 883   SetDevicePathNodeLength (&Node->Header, sizeof (NVME_NAMESPACE_DEVICE_PATH));
 884   Node->NamespaceId    = NamespaceId;
 885   Node->NamespaceUuid  = NamespaceUuid;
 886
 887   *DevicePath = (EFI_DEVICE_PATH_PROTOCOL *)Node;
 888   return EFI_SUCCESS;
 889 }
 890