MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.c

   1 /** @file
   2   NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows
   3   NVM Express specification.
   4
   5   Copyright (c) 2013 - 2017, Intel Corporation. All rights reserved.<BR>
   6   SPDX-License-Identifier: BSD-2-Clause-Patent
   7
   8 **/
   9
  10 #include "NvmExpress.h"
  11
  12 //
  13 // NVM Express Driver Binding Protocol Instance
  14 //
  15 EFI_DRIVER_BINDING_PROTOCOL  gNvmExpressDriverBinding = {
  16   NvmExpressDriverBindingSupported,
  17   NvmExpressDriverBindingStart,
  18   NvmExpressDriverBindingStop,
  19   0x10,
  20   NULL,
  21   NULL
  22 };
  23
  24 //
  25 // NVM Express EFI Driver Supported EFI Version Protocol Instance
  26 //
  27 EFI_DRIVER_SUPPORTED_EFI_VERSION_PROTOCOL  gNvmExpressDriverSupportedEfiVersion = {
  28   sizeof (EFI_DRIVER_SUPPORTED_EFI_VERSION_PROTOCOL), // Size of Protocol structure.
  29   0                                                   // Version number to be filled at start up.
  30 };
  31
  32 //
  33 // Template for NVM Express Pass Thru Mode data structure.
  34 //
  35 GLOBAL_REMOVE_IF_UNREFERENCED EFI_NVM_EXPRESS_PASS_THRU_MODE  gEfiNvmExpressPassThruMode = {
  36   EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL   |
  37   EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL    |
  38   EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_NONBLOCKIO |
  39   EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_CMD_SET_NVM,
  40   sizeof (UINTN),
  41   0x10100
  42 };
  43
  44 /**
  45   Check if the specified Nvm Express device namespace is active, and create child handles
  46   for them with BlockIo and DiskInfo protocol instances.
  47
  48   @param[in] Private         The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
  49   @param[in] NamespaceId     The NVM Express namespace ID  for which a device path node is to be
  50                              allocated and built. Caller must set the NamespaceId to zero if the
  51                              device path node will contain a valid UUID.
  52
  53   @retval EFI_SUCCESS        All the namespaces in the device are successfully enumerated.
  54   @return Others             Some error occurs when enumerating the namespaces.
  55
  56 **/
  57 EFI_STATUS
  58 EnumerateNvmeDevNamespace (
  59   IN NVME_CONTROLLER_PRIVATE_DATA  *Private,
  60   UINT32                           NamespaceId
  61   )
  62 {
  63   NVME_ADMIN_NAMESPACE_DATA  *NamespaceData;
  64   EFI_DEVICE_PATH_PROTOCOL   *NewDevicePathNode;
  65   EFI_DEVICE_PATH_PROTOCOL   *DevicePath;
  66   EFI_HANDLE                 DeviceHandle;
  67   EFI_DEVICE_PATH_PROTOCOL   *ParentDevicePath;
  68   EFI_DEVICE_PATH_PROTOCOL   *RemainingDevicePath;
  69   NVME_DEVICE_PRIVATE_DATA   *Device;
  70   EFI_STATUS                 Status;
  71   UINT32                     Lbads;
  72   UINT32                     Flbas;
  73   UINT32                     LbaFmtIdx;
  74   UINT8                      Sn[21];
  75   UINT8                      Mn[41];
  76   VOID                       *DummyInterface;
  77
  78   NewDevicePathNode = NULL;
  79   DevicePath        = NULL;
  80   Device            = NULL;
  81
  82   //
  83   // Allocate a buffer for Identify Namespace data
  84   //
  85   NamespaceData = AllocateZeroPool (sizeof (NVME_ADMIN_NAMESPACE_DATA));
  86   if (NamespaceData == NULL) {
  87     return EFI_OUT_OF_RESOURCES;
  88   }
  89
  90   ParentDevicePath = Private->ParentDevicePath;
  91   //
  92   // Identify Namespace
  93   //
  94   Status = NvmeIdentifyNamespace (
  95              Private,
  96              NamespaceId,
  97              (VOID *)NamespaceData
  98              );
  99   if (EFI_ERROR (Status)) {
 100     goto Exit;
 101   }
 102
 103   //
 104   // Validate Namespace
 105   //
 106   if (NamespaceData->Ncap == 0) {
 107     Status = EFI_DEVICE_ERROR;
 108   } else {
 109     //
 110     // allocate device private data for each discovered namespace
 111     //
 112     Device = AllocateZeroPool (sizeof (NVME_DEVICE_PRIVATE_DATA));
 113     if (Device == NULL) {
 114       Status = EFI_OUT_OF_RESOURCES;
 115       goto Exit;
 116     }
 117
 118     //
 119     // Initialize SSD namespace instance data
 120     //
 121     Device->Signature     = NVME_DEVICE_PRIVATE_DATA_SIGNATURE;
 122     Device->NamespaceId   = NamespaceId;
 123     Device->NamespaceUuid = NamespaceData->Eui64;
 124
 125     Device->ControllerHandle    = Private->ControllerHandle;
 126     Device->DriverBindingHandle = Private->DriverBindingHandle;
 127     Device->Controller          = Private;
 128
 129     //
 130     // Build BlockIo media structure
 131     //
 132     Device->Media.MediaId          = 0;
 133     Device->Media.RemovableMedia   = FALSE;
 134     Device->Media.MediaPresent     = TRUE;
 135     Device->Media.LogicalPartition = FALSE;
 136     Device->Media.ReadOnly         = FALSE;
 137     Device->Media.WriteCaching     = FALSE;
 138     Device->Media.IoAlign          = Private->PassThruMode.IoAlign;
 139
 140     Flbas                   = NamespaceData->Flbas;
 141     LbaFmtIdx               = Flbas & 0xF;
 142     Lbads                   = NamespaceData->LbaFormat[LbaFmtIdx].Lbads;
 143     Device->Media.BlockSize = (UINT32)1 << Lbads;
 144
 145     Device->Media.LastBlock                     = NamespaceData->Nsze - 1;
 146     Device->Media.LogicalBlocksPerPhysicalBlock = 1;
 147     Device->Media.LowestAlignedLba              = 1;
 148
 149     //
 150     // Create BlockIo Protocol instance
 151     //
 152     Device->BlockIo.Revision    = EFI_BLOCK_IO_PROTOCOL_REVISION2;
 153     Device->BlockIo.Media       = &Device->Media;
 154     Device->BlockIo.Reset       = NvmeBlockIoReset;
 155     Device->BlockIo.ReadBlocks  = NvmeBlockIoReadBlocks;
 156     Device->BlockIo.WriteBlocks = NvmeBlockIoWriteBlocks;
 157     Device->BlockIo.FlushBlocks = NvmeBlockIoFlushBlocks;
 158
 159     //
 160     // Create BlockIo2 Protocol instance
 161     //
 162     Device->BlockIo2.Media         = &Device->Media;
 163     Device->BlockIo2.Reset         = NvmeBlockIoResetEx;
 164     Device->BlockIo2.ReadBlocksEx  = NvmeBlockIoReadBlocksEx;
 165     Device->BlockIo2.WriteBlocksEx = NvmeBlockIoWriteBlocksEx;
 166     Device->BlockIo2.FlushBlocksEx = NvmeBlockIoFlushBlocksEx;
 167     InitializeListHead (&Device->AsyncQueue);
 168
 169     //
 170     // Create StorageSecurityProtocol Instance
 171     //
 172     Device->StorageSecurity.ReceiveData = NvmeStorageSecurityReceiveData;
 173     Device->StorageSecurity.SendData    = NvmeStorageSecuritySendData;
 174
 175     //
 176     // Create DiskInfo Protocol instance
 177     //
 178     CopyMem (&Device->NamespaceData, NamespaceData, sizeof (NVME_ADMIN_NAMESPACE_DATA));
 179     InitializeDiskInfo (Device);
 180
 181     //
 182     // Create a Nvm Express Namespace Device Path Node
 183     //
 184     Status = Private->Passthru.BuildDevicePath (
 185                                  &Private->Passthru,
 186                                  Device->NamespaceId,
 187                                  &NewDevicePathNode
 188                                  );
 189
 190     if (EFI_ERROR (Status)) {
 191       goto Exit;
 192     }
 193
 194     //
 195     // Append the SSD node to the controller's device path
 196     //
 197     DevicePath = AppendDevicePathNode (ParentDevicePath, NewDevicePathNode);
 198     if (DevicePath == NULL) {
 199       Status = EFI_OUT_OF_RESOURCES;
 200       goto Exit;
 201     }
 202
 203     DeviceHandle        = NULL;
 204     RemainingDevicePath = DevicePath;
 205     Status              = gBS->LocateDevicePath (&gEfiDevicePathProtocolGuid, &RemainingDevicePath, &DeviceHandle);
 206     if (!EFI_ERROR (Status) && (DeviceHandle != NULL) && IsDevicePathEnd (RemainingDevicePath)) {
 207       Status = EFI_ALREADY_STARTED;
 208       FreePool (DevicePath);
 209       goto Exit;
 210     }
 211
 212     Device->DevicePath = DevicePath;
 213
 214     //
 215     // Make sure the handle is NULL so we create a new handle
 216     //
 217     Device->DeviceHandle = NULL;
 218
 219     Status = gBS->InstallMultipleProtocolInterfaces (
 220                     &Device->DeviceHandle,
 221                     &gEfiDevicePathProtocolGuid,
 222                     Device->DevicePath,
 223                     &gEfiBlockIoProtocolGuid,
 224                     &Device->BlockIo,
 225                     &gEfiBlockIo2ProtocolGuid,
 226                     &Device->BlockIo2,
 227                     &gEfiDiskInfoProtocolGuid,
 228                     &Device->DiskInfo,
 229                     NULL
 230                     );
 231
 232     if (EFI_ERROR (Status)) {
 233       goto Exit;
 234     }
 235
 236     //
 237     // Check if the NVMe controller supports the Security Send and Security Receive commands
 238     //
 239     if ((Private->ControllerData->Oacs & SECURITY_SEND_RECEIVE_SUPPORTED) != 0) {
 240       Status = gBS->InstallProtocolInterface (
 241                       &Device->DeviceHandle,
 242                       &gEfiStorageSecurityCommandProtocolGuid,
 243                       EFI_NATIVE_INTERFACE,
 244                       &Device->StorageSecurity
 245                       );
 246       if (EFI_ERROR (Status)) {
 247         gBS->UninstallMultipleProtocolInterfaces (
 248                Device->DeviceHandle,
 249                &gEfiDevicePathProtocolGuid,
 250                Device->DevicePath,
 251                &gEfiBlockIoProtocolGuid,
 252                &Device->BlockIo,
 253                &gEfiBlockIo2ProtocolGuid,
 254                &Device->BlockIo2,
 255                &gEfiDiskInfoProtocolGuid,
 256                &Device->DiskInfo,
 257                NULL
 258                );
 259         goto Exit;
 260       }
 261     }
 262
 263     gBS->OpenProtocol (
 264            Private->ControllerHandle,
 265            &gEfiNvmExpressPassThruProtocolGuid,
 266            (VOID **)&DummyInterface,
 267            Private->DriverBindingHandle,
 268            Device->DeviceHandle,
 269            EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER
 270            );
 271
 272     //
 273     // Dump NvmExpress Identify Namespace Data
 274     //
 275     DEBUG ((DEBUG_INFO, " == NVME IDENTIFY NAMESPACE [%d] DATA ==\n", NamespaceId));
 276     DEBUG ((DEBUG_INFO, "    NSZE        : 0x%x\n", NamespaceData->Nsze));
 277     DEBUG ((DEBUG_INFO, "    NCAP        : 0x%x\n", NamespaceData->Ncap));
 278     DEBUG ((DEBUG_INFO, "    NUSE        : 0x%x\n", NamespaceData->Nuse));
 279     DEBUG ((DEBUG_INFO, "    LBAF0.LBADS : 0x%x\n", (NamespaceData->LbaFormat[0].Lbads)));
 280
 281     //
 282     // Build controller name for Component Name (2) protocol.
 283     //
 284     CopyMem (Sn, Private->ControllerData->Sn, sizeof (Private->ControllerData->Sn));
 285     Sn[20] = 0;
 286     CopyMem (Mn, Private->ControllerData->Mn, sizeof (Private->ControllerData->Mn));
 287     Mn[40] = 0;
 288     UnicodeSPrintAsciiFormat (Device->ModelName, sizeof (Device->ModelName), "%a-%a-%x", Sn, Mn, NamespaceData->Eui64);
 289
 290     AddUnicodeString2 (
 291       "eng",
 292       gNvmExpressComponentName.SupportedLanguages,
 293       &Device->ControllerNameTable,
 294       Device->ModelName,
 295       TRUE
 296       );
 297
 298     AddUnicodeString2 (
 299       "en",
 300       gNvmExpressComponentName2.SupportedLanguages,
 301       &Device->ControllerNameTable,
 302       Device->ModelName,
 303       FALSE
 304       );
 305   }
 306
 307 Exit:
 308   if (NamespaceData != NULL) {
 309     FreePool (NamespaceData);
 310   }
 311
 312   if (NewDevicePathNode != NULL) {
 313     FreePool (NewDevicePathNode);
 314   }
 315
 316   if (EFI_ERROR (Status) && (Device != NULL) && (Device->DevicePath != NULL)) {
 317     FreePool (Device->DevicePath);
 318   }
 319
 320   if (EFI_ERROR (Status) && (Device != NULL)) {
 321     FreePool (Device);
 322   }
 323
 324   return Status;
 325 }
 326
 327 /**
 328   Discover all Nvm Express device namespaces, and create child handles for them with BlockIo
 329   and DiskInfo protocol instances.
 330
 331   @param[in] Private         The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 332
 333   @retval EFI_SUCCESS        All the namespaces in the device are successfully enumerated.
 334   @return Others             Some error occurs when enumerating the namespaces.
 335
 336 **/
 337 EFI_STATUS
 338 DiscoverAllNamespaces (
 339   IN NVME_CONTROLLER_PRIVATE_DATA  *Private
 340   )
 341 {
 342   EFI_STATUS                          Status;
 343   UINT32                              NamespaceId;
 344   EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL  *Passthru;
 345
 346   NamespaceId = 0xFFFFFFFF;
 347   Passthru    = &Private->Passthru;
 348
 349   while (TRUE) {
 350     Status = Passthru->GetNextNamespace (
 351                          Passthru,
 352                          (UINT32 *)&NamespaceId
 353                          );
 354
 355     if (EFI_ERROR (Status)) {
 356       break;
 357     }
 358
 359     Status = EnumerateNvmeDevNamespace (
 360                Private,
 361                NamespaceId
 362                );
 363
 364     if (EFI_ERROR (Status)) {
 365       continue;
 366     }
 367   }
 368
 369   return EFI_SUCCESS;
 370 }
 371
 372 /**
 373   Unregisters a Nvm Express device namespace.
 374
 375   This function removes the protocols installed on the controller handle and
 376   frees the resources allocated for the namespace.
 377
 378   @param  This                  The pointer to EFI_DRIVER_BINDING_PROTOCOL instance.
 379   @param  Controller            The controller handle of the namespace.
 380   @param  Handle                The child handle.
 381
 382   @retval EFI_SUCCESS           The namespace is successfully unregistered.
 383   @return Others                Some error occurs when unregistering the namespace.
 384
 385 **/
 386 EFI_STATUS
 387 UnregisterNvmeNamespace (
 388   IN  EFI_DRIVER_BINDING_PROTOCOL  *This,
 389   IN  EFI_HANDLE                   Controller,
 390   IN  EFI_HANDLE                   Handle
 391   )
 392 {
 393   EFI_STATUS                             Status;
 394   EFI_BLOCK_IO_PROTOCOL                  *BlockIo;
 395   NVME_DEVICE_PRIVATE_DATA               *Device;
 396   EFI_STORAGE_SECURITY_COMMAND_PROTOCOL  *StorageSecurity;
 397   BOOLEAN                                IsEmpty;
 398   EFI_TPL                                OldTpl;
 399   VOID                                   *DummyInterface;
 400
 401   BlockIo = NULL;
 402
 403   Status = gBS->OpenProtocol (
 404                   Handle,
 405                   &gEfiBlockIoProtocolGuid,
 406                   (VOID **)&BlockIo,
 407                   This->DriverBindingHandle,
 408                   Controller,
 409                   EFI_OPEN_PROTOCOL_GET_PROTOCOL
 410                   );
 411   if (EFI_ERROR (Status)) {
 412     return Status;
 413   }
 414
 415   Device = NVME_DEVICE_PRIVATE_DATA_FROM_BLOCK_IO (BlockIo);
 416
 417   //
 418   // Wait for the device's asynchronous I/O queue to become empty.
 419   //
 420   while (TRUE) {
 421     OldTpl  = gBS->RaiseTPL (TPL_NOTIFY);
 422     IsEmpty = IsListEmpty (&Device->AsyncQueue);
 423     gBS->RestoreTPL (OldTpl);
 424
 425     if (IsEmpty) {
 426       break;
 427     }
 428
 429     gBS->Stall (100);
 430   }
 431
 432   //
 433   // Close the child handle
 434   //
 435   gBS->CloseProtocol (
 436          Controller,
 437          &gEfiNvmExpressPassThruProtocolGuid,
 438          This->DriverBindingHandle,
 439          Handle
 440          );
 441
 442   //
 443   // The Nvm Express driver installs the BlockIo and DiskInfo in the DriverBindingStart().
 444   // Here should uninstall both of them.
 445   //
 446   Status = gBS->UninstallMultipleProtocolInterfaces (
 447                   Handle,
 448                   &gEfiDevicePathProtocolGuid,
 449                   Device->DevicePath,
 450                   &gEfiBlockIoProtocolGuid,
 451                   &Device->BlockIo,
 452                   &gEfiBlockIo2ProtocolGuid,
 453                   &Device->BlockIo2,
 454                   &gEfiDiskInfoProtocolGuid,
 455                   &Device->DiskInfo,
 456                   NULL
 457                   );
 458
 459   if (EFI_ERROR (Status)) {
 460     gBS->OpenProtocol (
 461            Controller,
 462            &gEfiNvmExpressPassThruProtocolGuid,
 463            (VOID **)&DummyInterface,
 464            This->DriverBindingHandle,
 465            Handle,
 466            EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER
 467            );
 468     return Status;
 469   }
 470
 471   //
 472   // If Storage Security Command Protocol is installed, then uninstall this protocol.
 473   //
 474   Status = gBS->OpenProtocol (
 475                   Handle,
 476                   &gEfiStorageSecurityCommandProtocolGuid,
 477                   (VOID **)&StorageSecurity,
 478                   This->DriverBindingHandle,
 479                   Controller,
 480                   EFI_OPEN_PROTOCOL_GET_PROTOCOL
 481                   );
 482
 483   if (!EFI_ERROR (Status)) {
 484     Status = gBS->UninstallProtocolInterface (
 485                     Handle,
 486                     &gEfiStorageSecurityCommandProtocolGuid,
 487                     &Device->StorageSecurity
 488                     );
 489     if (EFI_ERROR (Status)) {
 490       gBS->OpenProtocol (
 491              Controller,
 492              &gEfiNvmExpressPassThruProtocolGuid,
 493              (VOID **)&DummyInterface,
 494              This->DriverBindingHandle,
 495              Handle,
 496              EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER
 497              );
 498       return Status;
 499     }
 500   }
 501
 502   if (Device->DevicePath != NULL) {
 503     FreePool (Device->DevicePath);
 504   }
 505
 506   if (Device->ControllerNameTable != NULL) {
 507     FreeUnicodeStringTable (Device->ControllerNameTable);
 508   }
 509
 510   FreePool (Device);
 511
 512   return EFI_SUCCESS;
 513 }
 514
 515 /**
 516   Call back function when the timer event is signaled.
 517
 518   @param[in]  Event     The Event this notify function registered to.
 519   @param[in]  Context   Pointer to the context data registered to the
 520                         Event.
 521
 522 **/
 523 VOID
 524 EFIAPI
 525 ProcessAsyncTaskList (
 526   IN EFI_EVENT  Event,
 527   IN VOID       *Context
 528   )
 529 {
 530   NVME_CONTROLLER_PRIVATE_DATA  *Private;
 531   EFI_PCI_IO_PROTOCOL           *PciIo;
 532   NVME_CQ                       *Cq;
 533   UINT16                        QueueId;
 534   UINT32                        Data;
 535   LIST_ENTRY                    *Link;
 536   LIST_ENTRY                    *NextLink;
 537   NVME_PASS_THRU_ASYNC_REQ      *AsyncRequest;
 538   NVME_BLKIO2_SUBTASK           *Subtask;
 539   NVME_BLKIO2_REQUEST           *BlkIo2Request;
 540   EFI_BLOCK_IO2_TOKEN           *Token;
 541   BOOLEAN                       HasNewItem;
 542   EFI_STATUS                    Status;
 543
 544   Private    = (NVME_CONTROLLER_PRIVATE_DATA *)Context;
 545   QueueId    = 2;
 546   Cq         = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
 547   HasNewItem = FALSE;
 548   PciIo      = Private->PciIo;
 549
 550   //
 551   // Submit asynchronous subtasks to the NVMe Submission Queue
 552   //
 553   for (Link = GetFirstNode (&Private->UnsubmittedSubtasks);
 554        !IsNull (&Private->UnsubmittedSubtasks, Link);
 555        Link = NextLink)
 556   {
 557     NextLink      = GetNextNode (&Private->UnsubmittedSubtasks, Link);
 558     Subtask       = NVME_BLKIO2_SUBTASK_FROM_LINK (Link);
 559     BlkIo2Request = Subtask->BlockIo2Request;
 560     Token         = BlkIo2Request->Token;
 561     RemoveEntryList (Link);
 562     BlkIo2Request->UnsubmittedSubtaskNum--;
 563
 564     //
 565     // If any previous subtask fails, do not process subsequent ones.
 566     //
 567     if (Token->TransactionStatus != EFI_SUCCESS) {
 568       if (IsListEmpty (&BlkIo2Request->SubtasksQueue) &&
 569           BlkIo2Request->LastSubtaskSubmitted &&
 570           (BlkIo2Request->UnsubmittedSubtaskNum == 0))
 571       {
 572         //
 573         // Remove the BlockIo2 request from the device asynchronous queue.
 574         //
 575         RemoveEntryList (&BlkIo2Request->Link);
 576         FreePool (BlkIo2Request);
 577         gBS->SignalEvent (Token->Event);
 578       }
 579
 580       FreePool (Subtask->CommandPacket->NvmeCmd);
 581       FreePool (Subtask->CommandPacket->NvmeCompletion);
 582       FreePool (Subtask->CommandPacket);
 583       FreePool (Subtask);
 584
 585       continue;
 586     }
 587
 588     Status = Private->Passthru.PassThru (
 589                                  &Private->Passthru,
 590                                  Subtask->NamespaceId,
 591                                  Subtask->CommandPacket,
 592                                  Subtask->Event
 593                                  );
 594     if (Status == EFI_NOT_READY) {
 595       InsertHeadList (&Private->UnsubmittedSubtasks, Link);
 596       BlkIo2Request->UnsubmittedSubtaskNum++;
 597       break;
 598     } else if (EFI_ERROR (Status)) {
 599       Token->TransactionStatus = EFI_DEVICE_ERROR;
 600
 601       if (IsListEmpty (&BlkIo2Request->SubtasksQueue) &&
 602           Subtask->IsLast)
 603       {
 604         //
 605         // Remove the BlockIo2 request from the device asynchronous queue.
 606         //
 607         RemoveEntryList (&BlkIo2Request->Link);
 608         FreePool (BlkIo2Request);
 609         gBS->SignalEvent (Token->Event);
 610       }
 611
 612       FreePool (Subtask->CommandPacket->NvmeCmd);
 613       FreePool (Subtask->CommandPacket->NvmeCompletion);
 614       FreePool (Subtask->CommandPacket);
 615       FreePool (Subtask);
 616     } else {
 617       InsertTailList (&BlkIo2Request->SubtasksQueue, Link);
 618       if (Subtask->IsLast) {
 619         BlkIo2Request->LastSubtaskSubmitted = TRUE;
 620       }
 621     }
 622   }
 623
 624   while (Cq->Pt != Private->Pt[QueueId]) {
 625     ASSERT (Cq->Sqid == QueueId);
 626
 627     HasNewItem = TRUE;
 628
 629     //
 630     // Find the command with given Command Id.
 631     //
 632     for (Link = GetFirstNode (&Private->AsyncPassThruQueue);
 633          !IsNull (&Private->AsyncPassThruQueue, Link);
 634          Link = NextLink)
 635     {
 636       NextLink     = GetNextNode (&Private->AsyncPassThruQueue, Link);
 637       AsyncRequest = NVME_PASS_THRU_ASYNC_REQ_FROM_THIS (Link);
 638       if (AsyncRequest->CommandId == Cq->Cid) {
 639         //
 640         // Copy the Respose Queue entry for this command to the callers
 641         // response buffer.
 642         //
 643         CopyMem (
 644           AsyncRequest->Packet->NvmeCompletion,
 645           Cq,
 646           sizeof (EFI_NVM_EXPRESS_COMPLETION)
 647           );
 648
 649         //
 650         // Free the resources allocated before cmd submission
 651         //
 652         if (AsyncRequest->MapData != NULL) {
 653           PciIo->Unmap (PciIo, AsyncRequest->MapData);
 654         }
 655
 656         if (AsyncRequest->MapMeta != NULL) {
 657           PciIo->Unmap (PciIo, AsyncRequest->MapMeta);
 658         }
 659
 660         if (AsyncRequest->MapPrpList != NULL) {
 661           PciIo->Unmap (PciIo, AsyncRequest->MapPrpList);
 662         }
 663
 664         if (AsyncRequest->PrpListHost != NULL) {
 665           PciIo->FreeBuffer (
 666                    PciIo,
 667                    AsyncRequest->PrpListNo,
 668                    AsyncRequest->PrpListHost
 669                    );
 670         }
 671
 672         RemoveEntryList (Link);
 673         gBS->SignalEvent (AsyncRequest->CallerEvent);
 674         FreePool (AsyncRequest);
 675
 676         //
 677         // Update submission queue head.
 678         //
 679         Private->AsyncSqHead = Cq->Sqhd;
 680         break;
 681       }
 682     }
 683
 684     Private->CqHdbl[QueueId].Cqh++;
 685     if (Private->CqHdbl[QueueId].Cqh > MIN (NVME_ASYNC_CCQ_SIZE, Private->Cap.Mqes)) {
 686       Private->CqHdbl[QueueId].Cqh = 0;
 687       Private->Pt[QueueId]        ^= 1;
 688     }
 689
 690     Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
 691   }
 692
 693   if (HasNewItem) {
 694     Data = ReadUnaligned32 ((UINT32 *)&Private->CqHdbl[QueueId]);
 695     PciIo->Mem.Write (
 696                  PciIo,
 697                  EfiPciIoWidthUint32,
 698                  NVME_BAR,
 699                  NVME_CQHDBL_OFFSET (QueueId, Private->Cap.Dstrd),
 700                  1,
 701                  &Data
 702                  );
 703   }
 704 }
 705
 706 /**
 707   Tests to see if this driver supports a given controller. If a child device is provided,
 708   it further tests to see if this driver supports creating a handle for the specified child device.
 709
 710   This function checks to see if the driver specified by This supports the device specified by
 711   ControllerHandle. Drivers will typically use the device path attached to
 712   ControllerHandle and/or the services from the bus I/O abstraction attached to
 713   ControllerHandle to determine if the driver supports ControllerHandle. This function
 714   may be called many times during platform initialization. In order to reduce boot times, the tests
 715   performed by this function must be very small, and take as little time as possible to execute. This
 716   function must not change the state of any hardware devices, and this function must be aware that the
 717   device specified by ControllerHandle may already be managed by the same driver or a
 718   different driver. This function must match its calls to AllocatePages() with FreePages(),
 719   AllocatePool() with FreePool(), and OpenProtocol() with CloseProtocol().
 720   Since ControllerHandle may have been previously started by the same driver, if a protocol is
 721   already in the opened state, then it must not be closed with CloseProtocol(). This is required
 722   to guarantee the state of ControllerHandle is not modified by this function.
 723
 724   @param[in]  This                 A pointer to the EFI_DRIVER_BINDING_PROTOCOL instance.
 725   @param[in]  ControllerHandle     The handle of the controller to test. This handle
 726                                    must support a protocol interface that supplies
 727                                    an I/O abstraction to the driver.
 728   @param[in]  RemainingDevicePath  A pointer to the remaining portion of a device path.  This
 729                                    parameter is ignored by device drivers, and is optional for bus
 730                                    drivers. For bus drivers, if this parameter is not NULL, then
 731                                    the bus driver must determine if the bus controller specified
 732                                    by ControllerHandle and the child controller specified
 733                                    by RemainingDevicePath are both supported by this
 734                                    bus driver.
 735
 736   @retval EFI_SUCCESS              The device specified by ControllerHandle and
 737                                    RemainingDevicePath is supported by the driver specified by This.
 738   @retval EFI_ALREADY_STARTED      The device specified by ControllerHandle and
 739                                    RemainingDevicePath is already being managed by the driver
 740                                    specified by This.
 741   @retval EFI_ACCESS_DENIED        The device specified by ControllerHandle and
 742                                    RemainingDevicePath is already being managed by a different
 743                                    driver or an application that requires exclusive access.
 744                                    Currently not implemented.
 745   @retval EFI_UNSUPPORTED          The device specified by ControllerHandle and
 746                                    RemainingDevicePath is not supported by the driver specified by This.
 747 **/
 748 EFI_STATUS
 749 EFIAPI
 750 NvmExpressDriverBindingSupported (
 751   IN EFI_DRIVER_BINDING_PROTOCOL  *This,
 752   IN EFI_HANDLE                   Controller,
 753   IN EFI_DEVICE_PATH_PROTOCOL     *RemainingDevicePath
 754   )
 755 {
 756   EFI_STATUS                Status;
 757   EFI_DEV_PATH_PTR          DevicePathNode;
 758   EFI_DEVICE_PATH_PROTOCOL  *ParentDevicePath;
 759   EFI_PCI_IO_PROTOCOL       *PciIo;
 760   UINT8                     ClassCode[3];
 761
 762   //
 763   // Check whether device path is valid
 764   //
 765   if (RemainingDevicePath != NULL) {
 766     //
 767     // Check if RemainingDevicePath is the End of Device Path Node,
 768     // if yes, go on checking other conditions
 769     //
 770     if (!IsDevicePathEnd (RemainingDevicePath)) {
 771       //
 772       // If RemainingDevicePath isn't the End of Device Path Node,
 773       // check its validation
 774       //
 775       DevicePathNode.DevPath = RemainingDevicePath;
 776
 777       if ((DevicePathNode.DevPath->Type    != MESSAGING_DEVICE_PATH) ||
 778           (DevicePathNode.DevPath->SubType != MSG_NVME_NAMESPACE_DP) ||
 779           (DevicePathNodeLength (DevicePathNode.DevPath) != sizeof (NVME_NAMESPACE_DEVICE_PATH)))
 780       {
 781         return EFI_UNSUPPORTED;
 782       }
 783     }
 784   }
 785
 786   //
 787   // Open the EFI Device Path protocol needed to perform the supported test
 788   //
 789   Status = gBS->OpenProtocol (
 790                   Controller,
 791                   &gEfiDevicePathProtocolGuid,
 792                   (VOID **)&ParentDevicePath,
 793                   This->DriverBindingHandle,
 794                   Controller,
 795                   EFI_OPEN_PROTOCOL_BY_DRIVER
 796                   );
 797   if (Status == EFI_ALREADY_STARTED) {
 798     return EFI_SUCCESS;
 799   }
 800
 801   if (EFI_ERROR (Status)) {
 802     return Status;
 803   }
 804
 805   //
 806   // Close protocol, don't use device path protocol in the Support() function
 807   //
 808   gBS->CloseProtocol (
 809          Controller,
 810          &gEfiDevicePathProtocolGuid,
 811          This->DriverBindingHandle,
 812          Controller
 813          );
 814
 815   //
 816   // Attempt to Open PCI I/O Protocol
 817   //
 818   Status = gBS->OpenProtocol (
 819                   Controller,
 820                   &gEfiPciIoProtocolGuid,
 821                   (VOID **)&PciIo,
 822                   This->DriverBindingHandle,
 823                   Controller,
 824                   EFI_OPEN_PROTOCOL_BY_DRIVER
 825                   );
 826   if (Status == EFI_ALREADY_STARTED) {
 827     return EFI_SUCCESS;
 828   }
 829
 830   if (EFI_ERROR (Status)) {
 831     return Status;
 832   }
 833
 834   //
 835   // Now further check the PCI header: Base class (offset 0x0B) and Sub Class (offset 0x0A).
 836   // This controller should be a Nvm Express controller.
 837   //
 838   Status = PciIo->Pci.Read (
 839                         PciIo,
 840                         EfiPciIoWidthUint8,
 841                         PCI_CLASSCODE_OFFSET,
 842                         sizeof (ClassCode),
 843                         ClassCode
 844                         );
 845   if (EFI_ERROR (Status)) {
 846     goto Done;
 847   }
 848
 849   //
 850   // Examine Nvm Express controller PCI Configuration table fields
 851   //
 852   if ((ClassCode[0] != PCI_IF_NVMHCI) || (ClassCode[1] != PCI_CLASS_MASS_STORAGE_NVM) || (ClassCode[2] != PCI_CLASS_MASS_STORAGE)) {
 853     Status = EFI_UNSUPPORTED;
 854   }
 855
 856 Done:
 857   gBS->CloseProtocol (
 858          Controller,
 859          &gEfiPciIoProtocolGuid,
 860          This->DriverBindingHandle,
 861          Controller
 862          );
 863
 864   return Status;
 865 }
 866
 867 /**
 868   Starts a device controller or a bus controller.
 869
 870   The Start() function is designed to be invoked from the EFI boot service ConnectController().
 871   As a result, much of the error checking on the parameters to Start() has been moved into this
 872   common boot service. It is legal to call Start() from other locations,
 873   but the following calling restrictions must be followed or the system behavior will not be deterministic.
 874   1. ControllerHandle must be a valid EFI_HANDLE.
 875   2. If RemainingDevicePath is not NULL, then it must be a pointer to a naturally aligned
 876      EFI_DEVICE_PATH_PROTOCOL.
 877   3. Prior to calling Start(), the Supported() function for the driver specified by This must
 878      have been called with the same calling parameters, and Supported() must have returned EFI_SUCCESS.
 879
 880   @param[in]  This                 A pointer to the EFI_DRIVER_BINDING_PROTOCOL instance.
 881   @param[in]  ControllerHandle     The handle of the controller to start. This handle
 882                                    must support a protocol interface that supplies
 883                                    an I/O abstraction to the driver.
 884   @param[in]  RemainingDevicePath  A pointer to the remaining portion of a device path.  This
 885                                    parameter is ignored by device drivers, and is optional for bus
 886                                    drivers. For a bus driver, if this parameter is NULL, then handles
 887                                    for all the children of Controller are created by this driver.
 888                                    If this parameter is not NULL and the first Device Path Node is
 889                                    not the End of Device Path Node, then only the handle for the
 890                                    child device specified by the first Device Path Node of
 891                                    RemainingDevicePath is created by this driver.
 892                                    If the first Device Path Node of RemainingDevicePath is
 893                                    the End of Device Path Node, no child handle is created by this
 894                                    driver.
 895
 896   @retval EFI_SUCCESS              The device was started.
 897   @retval EFI_DEVICE_ERROR         The device could not be started due to a device error.Currently not implemented.
 898   @retval EFI_OUT_OF_RESOURCES     The request could not be completed due to a lack of resources.
 899   @retval Others                   The driver failded to start the device.
 900
 901 **/
 902 EFI_STATUS
 903 EFIAPI
 904 NvmExpressDriverBindingStart (
 905   IN EFI_DRIVER_BINDING_PROTOCOL  *This,
 906   IN EFI_HANDLE                   Controller,
 907   IN EFI_DEVICE_PATH_PROTOCOL     *RemainingDevicePath
 908   )
 909 {
 910   EFI_STATUS                          Status;
 911   EFI_PCI_IO_PROTOCOL                 *PciIo;
 912   NVME_CONTROLLER_PRIVATE_DATA        *Private;
 913   EFI_DEVICE_PATH_PROTOCOL            *ParentDevicePath;
 914   UINT32                              NamespaceId;
 915   EFI_PHYSICAL_ADDRESS                MappedAddr;
 916   UINTN                               Bytes;
 917   EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL  *Passthru;
 918
 919   DEBUG ((DEBUG_INFO, "NvmExpressDriverBindingStart: start\n"));
 920
 921   Private          = NULL;
 922   Passthru         = NULL;
 923   ParentDevicePath = NULL;
 924
 925   Status = gBS->OpenProtocol (
 926                   Controller,
 927                   &gEfiDevicePathProtocolGuid,
 928                   (VOID **)&ParentDevicePath,
 929                   This->DriverBindingHandle,
 930                   Controller,
 931                   EFI_OPEN_PROTOCOL_BY_DRIVER
 932                   );
 933   if ((EFI_ERROR (Status)) && (Status != EFI_ALREADY_STARTED)) {
 934     return Status;
 935   }
 936
 937   Status = gBS->OpenProtocol (
 938                   Controller,
 939                   &gEfiPciIoProtocolGuid,
 940                   (VOID **)&PciIo,
 941                   This->DriverBindingHandle,
 942                   Controller,
 943                   EFI_OPEN_PROTOCOL_BY_DRIVER
 944                   );
 945
 946   if (EFI_ERROR (Status) && (Status != EFI_ALREADY_STARTED)) {
 947     return Status;
 948   }
 949
 950   //
 951   // Check EFI_ALREADY_STARTED to reuse the original NVME_CONTROLLER_PRIVATE_DATA.
 952   //
 953   if (Status != EFI_ALREADY_STARTED) {
 954     Private = AllocateZeroPool (sizeof (NVME_CONTROLLER_PRIVATE_DATA));
 955
 956     if (Private == NULL) {
 957       DEBUG ((DEBUG_ERROR, "NvmExpressDriverBindingStart: allocating pool for Nvme Private Data failed!\n"));
 958       Status = EFI_OUT_OF_RESOURCES;
 959       goto Exit;
 960     }
 961
 962     //
 963     // Save original PCI attributes
 964     //
 965     Status = PciIo->Attributes (
 966                       PciIo,
 967                       EfiPciIoAttributeOperationGet,
 968                       0,
 969                       &Private->PciAttributes
 970                       );
 971
 972     if (EFI_ERROR (Status)) {
 973       return Status;
 974     }
 975
 976     //
 977     // Enable 64-bit DMA support in the PCI layer.
 978     //
 979     Status = PciIo->Attributes (
 980                       PciIo,
 981                       EfiPciIoAttributeOperationEnable,
 982                       EFI_PCI_IO_ATTRIBUTE_DUAL_ADDRESS_CYCLE,
 983                       NULL
 984                       );
 985     if (EFI_ERROR (Status)) {
 986       DEBUG ((DEBUG_WARN, "NvmExpressDriverBindingStart: failed to enable 64-bit DMA (%r)\n", Status));
 987     }
 988
 989     //
 990     // 6 x 4kB aligned buffers will be carved out of this buffer.
 991     // 1st 4kB boundary is the start of the admin submission queue.
 992     // 2nd 4kB boundary is the start of the admin completion queue.
 993     // 3rd 4kB boundary is the start of I/O submission queue #1.
 994     // 4th 4kB boundary is the start of I/O completion queue #1.
 995     // 5th 4kB boundary is the start of I/O submission queue #2.
 996     // 6th 4kB boundary is the start of I/O completion queue #2.
 997     //
 998     // Allocate 6 pages of memory, then map it for bus master read and write.
 999     //
1000     Status = PciIo->AllocateBuffer (
1001                       PciIo,
1002                       AllocateAnyPages,
1003                       EfiBootServicesData,
1004                       6,
1005                       (VOID **)&Private->Buffer,
1006                       0
1007                       );
1008     if (EFI_ERROR (Status)) {
1009       goto Exit;
1010     }
1011
1012     Bytes  = EFI_PAGES_TO_SIZE (6);
1013     Status = PciIo->Map (
1014                       PciIo,
1015                       EfiPciIoOperationBusMasterCommonBuffer,
1016                       Private->Buffer,
1017                       &Bytes,
1018                       &MappedAddr,
1019                       &Private->Mapping
1020                       );
1021
1022     if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (6))) {
1023       goto Exit;
1024     }
1025
1026     Private->BufferPciAddr = (UINT8 *)(UINTN)MappedAddr;
1027
1028     Private->Signature                 = NVME_CONTROLLER_PRIVATE_DATA_SIGNATURE;
1029     Private->ControllerHandle          = Controller;
1030     Private->ImageHandle               = This->DriverBindingHandle;
1031     Private->DriverBindingHandle       = This->DriverBindingHandle;
1032     Private->PciIo                     = PciIo;
1033     Private->ParentDevicePath          = ParentDevicePath;
1034     Private->Passthru.Mode             = &Private->PassThruMode;
1035     Private->Passthru.PassThru         = NvmExpressPassThru;
1036     Private->Passthru.GetNextNamespace = NvmExpressGetNextNamespace;
1037     Private->Passthru.BuildDevicePath  = NvmExpressBuildDevicePath;
1038     Private->Passthru.GetNamespace     = NvmExpressGetNamespace;
1039     CopyMem (&Private->PassThruMode, &gEfiNvmExpressPassThruMode, sizeof (EFI_NVM_EXPRESS_PASS_THRU_MODE));
1040     InitializeListHead (&Private->AsyncPassThruQueue);
1041     InitializeListHead (&Private->UnsubmittedSubtasks);
1042
1043     Status = NvmeControllerInit (Private);
1044     if (EFI_ERROR (Status)) {
1045       goto Exit;
1046     }
1047
1048     //
1049     // Start the asynchronous I/O completion monitor
1050     //
1051     Status = gBS->CreateEvent (
1052                     EVT_TIMER | EVT_NOTIFY_SIGNAL,
1053                     TPL_NOTIFY,
1054                     ProcessAsyncTaskList,
1055                     Private,
1056                     &Private->TimerEvent
1057                     );
1058     if (EFI_ERROR (Status)) {
1059       goto Exit;
1060     }
1061
1062     Status = gBS->SetTimer (
1063                     Private->TimerEvent,
1064                     TimerPeriodic,
1065                     NVME_HC_ASYNC_TIMER
1066                     );
1067     if (EFI_ERROR (Status)) {
1068       goto Exit;
1069     }
1070
1071     Status = gBS->InstallMultipleProtocolInterfaces (
1072                     &Controller,
1073                     &gEfiNvmExpressPassThruProtocolGuid,
1074                     &Private->Passthru,
1075                     NULL
1076                     );
1077     if (EFI_ERROR (Status)) {
1078       goto Exit;
1079     }
1080
1081     NvmeRegisterShutdownNotification ();
1082   } else {
1083     Status = gBS->OpenProtocol (
1084                     Controller,
1085                     &gEfiNvmExpressPassThruProtocolGuid,
1086                     (VOID **)&Passthru,
1087                     This->DriverBindingHandle,
1088                     Controller,
1089                     EFI_OPEN_PROTOCOL_GET_PROTOCOL
1090                     );
1091     if (EFI_ERROR (Status)) {
1092       goto Exit;
1093     }
1094
1095     Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (Passthru);
1096   }
1097
1098   if (RemainingDevicePath == NULL) {
1099     //
1100     // Enumerate all NVME namespaces in the controller
1101     //
1102     Status = DiscoverAllNamespaces (
1103                Private
1104                );
1105   } else if (!IsDevicePathEnd (RemainingDevicePath)) {
1106     //
1107     // Enumerate the specified NVME namespace
1108     //
1109     Status = Private->Passthru.GetNamespace (
1110                                  &Private->Passthru,
1111                                  RemainingDevicePath,
1112                                  &NamespaceId
1113                                  );
1114
1115     if (!EFI_ERROR (Status)) {
1116       Status = EnumerateNvmeDevNamespace (
1117                  Private,
1118                  NamespaceId
1119                  );
1120     }
1121   }
1122
1123   DEBUG ((DEBUG_INFO, "NvmExpressDriverBindingStart: end successfully\n"));
1124   return EFI_SUCCESS;
1125
1126 Exit:
1127   if ((Private != NULL) && (Private->Mapping != NULL)) {
1128     PciIo->Unmap (PciIo, Private->Mapping);
1129   }
1130
1131   if ((Private != NULL) && (Private->Buffer != NULL)) {
1132     PciIo->FreeBuffer (PciIo, 6, Private->Buffer);
1133   }
1134
1135   if ((Private != NULL) && (Private->ControllerData != NULL)) {
1136     FreePool (Private->ControllerData);
1137   }
1138
1139   if (Private != NULL) {
1140     if (Private->TimerEvent != NULL) {
1141       gBS->CloseEvent (Private->TimerEvent);
1142     }
1143
1144     FreePool (Private);
1145   }
1146
1147   gBS->CloseProtocol (
1148          Controller,
1149          &gEfiPciIoProtocolGuid,
1150          This->DriverBindingHandle,
1151          Controller
1152          );
1153
1154   gBS->CloseProtocol (
1155          Controller,
1156          &gEfiDevicePathProtocolGuid,
1157          This->DriverBindingHandle,
1158          Controller
1159          );
1160
1161   DEBUG ((DEBUG_INFO, "NvmExpressDriverBindingStart: end with %r\n", Status));
1162
1163   return Status;
1164 }
1165
1166 /**
1167   Stops a device controller or a bus controller.
1168
1169   The Stop() function is designed to be invoked from the EFI boot service DisconnectController().
1170   As a result, much of the error checking on the parameters to Stop() has been moved
1171   into this common boot service. It is legal to call Stop() from other locations,
1172   but the following calling restrictions must be followed or the system behavior will not be deterministic.
1173   1. ControllerHandle must be a valid EFI_HANDLE that was used on a previous call to this
1174      same driver's Start() function.
1175   2. The first NumberOfChildren handles of ChildHandleBuffer must all be a valid
1176      EFI_HANDLE. In addition, all of these handles must have been created in this driver's
1177      Start() function, and the Start() function must have called OpenProtocol() on
1178      ControllerHandle with an Attribute of EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER.
1179
1180   @param[in]  This              A pointer to the EFI_DRIVER_BINDING_PROTOCOL instance.
1181   @param[in]  ControllerHandle  A handle to the device being stopped. The handle must
1182                                 support a bus specific I/O protocol for the driver
1183                                 to use to stop the device.
1184   @param[in]  NumberOfChildren  The number of child device handles in ChildHandleBuffer.
1185   @param[in]  ChildHandleBuffer An array of child handles to be freed. May be NULL
1186                                 if NumberOfChildren is 0.
1187
1188   @retval EFI_SUCCESS           The device was stopped.
1189   @retval EFI_DEVICE_ERROR      The device could not be stopped due to a device error.
1190
1191 **/
1192 EFI_STATUS
1193 EFIAPI
1194 NvmExpressDriverBindingStop (
1195   IN  EFI_DRIVER_BINDING_PROTOCOL  *This,
1196   IN  EFI_HANDLE                   Controller,
1197   IN  UINTN                        NumberOfChildren,
1198   IN  EFI_HANDLE                   *ChildHandleBuffer
1199   )
1200 {
1201   EFI_STATUS                          Status;
1202   BOOLEAN                             AllChildrenStopped;
1203   UINTN                               Index;
1204   NVME_CONTROLLER_PRIVATE_DATA        *Private;
1205   EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL  *PassThru;
1206   BOOLEAN                             IsEmpty;
1207   EFI_TPL                             OldTpl;
1208
1209   if (NumberOfChildren == 0) {
1210     Status = gBS->OpenProtocol (
1211                     Controller,
1212                     &gEfiNvmExpressPassThruProtocolGuid,
1213                     (VOID **)&PassThru,
1214                     This->DriverBindingHandle,
1215                     Controller,
1216                     EFI_OPEN_PROTOCOL_GET_PROTOCOL
1217                     );
1218
1219     if (!EFI_ERROR (Status)) {
1220       Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (PassThru);
1221
1222       //
1223       // Wait for the asynchronous PassThru queue to become empty.
1224       //
1225       while (TRUE) {
1226         OldTpl  = gBS->RaiseTPL (TPL_NOTIFY);
1227         IsEmpty = IsListEmpty (&Private->AsyncPassThruQueue) &&
1228                   IsListEmpty (&Private->UnsubmittedSubtasks);
1229         gBS->RestoreTPL (OldTpl);
1230
1231         if (IsEmpty) {
1232           break;
1233         }
1234
1235         gBS->Stall (100);
1236       }
1237
1238       gBS->UninstallMultipleProtocolInterfaces (
1239              Controller,
1240              &gEfiNvmExpressPassThruProtocolGuid,
1241              PassThru,
1242              NULL
1243              );
1244
1245       if (Private->TimerEvent != NULL) {
1246         gBS->CloseEvent (Private->TimerEvent);
1247       }
1248
1249       if (Private->Mapping != NULL) {
1250         Private->PciIo->Unmap (Private->PciIo, Private->Mapping);
1251       }
1252
1253       if (Private->Buffer != NULL) {
1254         Private->PciIo->FreeBuffer (Private->PciIo, 6, Private->Buffer);
1255       }
1256
1257       FreePool (Private->ControllerData);
1258       FreePool (Private);
1259     }
1260
1261     gBS->CloseProtocol (
1262            Controller,
1263            &gEfiPciIoProtocolGuid,
1264            This->DriverBindingHandle,
1265            Controller
1266            );
1267     gBS->CloseProtocol (
1268            Controller,
1269            &gEfiDevicePathProtocolGuid,
1270            This->DriverBindingHandle,
1271            Controller
1272            );
1273
1274     NvmeUnregisterShutdownNotification ();
1275
1276     return EFI_SUCCESS;
1277   }
1278
1279   AllChildrenStopped = TRUE;
1280
1281   for (Index = 0; Index < NumberOfChildren; Index++) {
1282     Status = UnregisterNvmeNamespace (This, Controller, ChildHandleBuffer[Index]);
1283     if (EFI_ERROR (Status)) {
1284       AllChildrenStopped = FALSE;
1285     }
1286   }
1287
1288   if (!AllChildrenStopped) {
1289     return EFI_DEVICE_ERROR;
1290   }
1291
1292   return EFI_SUCCESS;
1293 }
1294
1295 /**
1296   This is the unload handle for the NVM Express driver.
1297
1298   Disconnect the driver specified by ImageHandle from the NVMe device in the handle database.
1299   Uninstall all the protocols installed in the driver.
1300
1301   @param[in]  ImageHandle       The drivers' driver image.
1302
1303   @retval EFI_SUCCESS           The image is unloaded.
1304   @retval Others                Failed to unload the image.
1305
1306 **/
1307 EFI_STATUS
1308 EFIAPI
1309 NvmExpressUnload (
1310   IN EFI_HANDLE  ImageHandle
1311   )
1312 {
1313   EFI_STATUS                    Status;
1314   EFI_HANDLE                    *DeviceHandleBuffer;
1315   UINTN                         DeviceHandleCount;
1316   UINTN                         Index;
1317   EFI_COMPONENT_NAME_PROTOCOL   *ComponentName;
1318   EFI_COMPONENT_NAME2_PROTOCOL  *ComponentName2;
1319
1320   //
1321   // Get the list of the device handles managed by this driver.
1322   // If there is an error getting the list, then means the driver
1323   // doesn't manage any device. At this way, we would only close
1324   // those protocols installed at image handle.
1325   //
1326   DeviceHandleBuffer = NULL;
1327   Status             = gBS->LocateHandleBuffer (
1328                               ByProtocol,
1329                               &gEfiNvmExpressPassThruProtocolGuid,
1330                               NULL,
1331                               &DeviceHandleCount,
1332                               &DeviceHandleBuffer
1333                               );
1334
1335   if (!EFI_ERROR (Status)) {
1336     //
1337     // Disconnect the driver specified by ImageHandle from all
1338     // the devices in the handle database.
1339     //
1340     for (Index = 0; Index < DeviceHandleCount; Index++) {
1341       Status = gBS->DisconnectController (
1342                       DeviceHandleBuffer[Index],
1343                       ImageHandle,
1344                       NULL
1345                       );
1346       if (EFI_ERROR (Status)) {
1347         goto EXIT;
1348       }
1349     }
1350   }
1351
1352   //
1353   // Uninstall all the protocols installed in the driver entry point
1354   //
1355   Status = gBS->UninstallMultipleProtocolInterfaces (
1356                   ImageHandle,
1357                   &gEfiDriverBindingProtocolGuid,
1358                   &gNvmExpressDriverBinding,
1359                   &gEfiDriverSupportedEfiVersionProtocolGuid,
1360                   &gNvmExpressDriverSupportedEfiVersion,
1361                   NULL
1362                   );
1363
1364   if (EFI_ERROR (Status)) {
1365     goto EXIT;
1366   }
1367
1368   //
1369   // Note we have to one by one uninstall the following protocols.
1370   // It's because some of them are optionally installed based on
1371   // the following PCD settings.
1372   //   gEfiMdePkgTokenSpaceGuid.PcdDriverDiagnosticsDisable
1373   //   gEfiMdePkgTokenSpaceGuid.PcdComponentNameDisable
1374   //   gEfiMdePkgTokenSpaceGuid.PcdDriverDiagnostics2Disable
1375   //   gEfiMdePkgTokenSpaceGuid.PcdComponentName2Disable
1376   //
1377   Status = gBS->HandleProtocol (
1378                   ImageHandle,
1379                   &gEfiComponentNameProtocolGuid,
1380                   (VOID **)&ComponentName
1381                   );
1382   if (!EFI_ERROR (Status)) {
1383     gBS->UninstallProtocolInterface (
1384            ImageHandle,
1385            &gEfiComponentNameProtocolGuid,
1386            ComponentName
1387            );
1388   }
1389
1390   Status = gBS->HandleProtocol (
1391                   ImageHandle,
1392                   &gEfiComponentName2ProtocolGuid,
1393                   (VOID **)&ComponentName2
1394                   );
1395   if (!EFI_ERROR (Status)) {
1396     gBS->UninstallProtocolInterface (
1397            ImageHandle,
1398            &gEfiComponentName2ProtocolGuid,
1399            ComponentName2
1400            );
1401   }
1402
1403   Status = EFI_SUCCESS;
1404
1405 EXIT:
1406   //
1407   // Free the buffer containing the list of handles from the handle database
1408   //
1409   if (DeviceHandleBuffer != NULL) {
1410     gBS->FreePool (DeviceHandleBuffer);
1411   }
1412
1413   return Status;
1414 }
1415
1416 /**
1417   The entry point for Nvm Express driver, used to install Nvm Express driver on the ImageHandle.
1418
1419   @param  ImageHandle   The firmware allocated handle for this driver image.
1420   @param  SystemTable   Pointer to the EFI system table.
1421
1422   @retval EFI_SUCCESS   Driver loaded.
1423   @retval other         Driver not loaded.
1424
1425 **/
1426 EFI_STATUS
1427 EFIAPI
1428 NvmExpressDriverEntry (
1429   IN EFI_HANDLE        ImageHandle,
1430   IN EFI_SYSTEM_TABLE  *SystemTable
1431   )
1432 {
1433   EFI_STATUS  Status;
1434
1435   Status = EfiLibInstallDriverBindingComponentName2 (
1436              ImageHandle,
1437              SystemTable,
1438              &gNvmExpressDriverBinding,
1439              ImageHandle,
1440              &gNvmExpressComponentName,
1441              &gNvmExpressComponentName2
1442              );
1443   ASSERT_EFI_ERROR (Status);
1444
1445   //
1446   // Install EFI Driver Supported EFI Version Protocol required for
1447   // EFI drivers that are on PCI and other plug in cards.
1448   //
1449   gNvmExpressDriverSupportedEfiVersion.FirmwareVersion = 0x00020028;
1450   Status                                               = gBS->InstallMultipleProtocolInterfaces (
1451                                                                 &ImageHandle,
1452                                                                 &gEfiDriverSupportedEfiVersionProtocolGuid,
1453                                                                 &gNvmExpressDriverSupportedEfiVersion,
1454                                                                 NULL
1455                                                                 );
1456   ASSERT_EFI_ERROR (Status);
1457   return Status;
1458 }