MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.c

   1 /** @file
   2   NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows
   3   NVM Express specification.
   4
   5   Copyright (c) 2013 - 2017, Intel Corporation. All rights reserved.<BR>
   6   SPDX-License-Identifier: BSD-2-Clause-Patent
   7
   8 **/
   9
  10 #include "NvmExpress.h"
  11
  12 //
  13 // NVM Express Driver Binding Protocol Instance
  14 //
  15 EFI_DRIVER_BINDING_PROTOCOL  gNvmExpressDriverBinding = {
  16   NvmExpressDriverBindingSupported,
  17   NvmExpressDriverBindingStart,
  18   NvmExpressDriverBindingStop,
  19   0x10,
  20   NULL,
  21   NULL
  22 };
  23
  24 //
  25 // NVM Express EFI Driver Supported EFI Version Protocol Instance
  26 //
  27 EFI_DRIVER_SUPPORTED_EFI_VERSION_PROTOCOL  gNvmExpressDriverSupportedEfiVersion = {
  28   sizeof (EFI_DRIVER_SUPPORTED_EFI_VERSION_PROTOCOL), // Size of Protocol structure.
  29   0                                                   // Version number to be filled at start up.
  30 };
  31
  32 //
  33 // Template for NVM Express Pass Thru Mode data structure.
  34 //
  35 GLOBAL_REMOVE_IF_UNREFERENCED EFI_NVM_EXPRESS_PASS_THRU_MODE  gEfiNvmExpressPassThruMode = {
  36   EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL   |
  37   EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL    |
  38   EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_NONBLOCKIO |
  39   EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_CMD_SET_NVM,
  40   sizeof (UINTN),
  41   0x10100
  42 };
  43
  44 /**
  45   Check if the specified Nvm Express device namespace is active, and create child handles
  46   for them with BlockIo and DiskInfo protocol instances.
  47
  48   @param[in] Private         The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
  49   @param[in] NamespaceId     The NVM Express namespace ID  for which a device path node is to be
  50                              allocated and built. Caller must set the NamespaceId to zero if the
  51                              device path node will contain a valid UUID.
  52
  53   @retval EFI_SUCCESS        All the namespaces in the device are successfully enumerated.
  54   @return Others             Some error occurs when enumerating the namespaces.
  55
  56 **/
  57 EFI_STATUS
  58 EnumerateNvmeDevNamespace (
  59   IN NVME_CONTROLLER_PRIVATE_DATA  *Private,
  60   UINT32                           NamespaceId
  61   )
  62 {
  63   NVME_ADMIN_NAMESPACE_DATA  *NamespaceData;
  64   EFI_DEVICE_PATH_PROTOCOL   *NewDevicePathNode;
  65   EFI_DEVICE_PATH_PROTOCOL   *DevicePath;
  66   EFI_HANDLE                 DeviceHandle;
  67   EFI_DEVICE_PATH_PROTOCOL   *ParentDevicePath;
  68   EFI_DEVICE_PATH_PROTOCOL   *RemainingDevicePath;
  69   NVME_DEVICE_PRIVATE_DATA   *Device;
  70   EFI_STATUS                 Status;
  71   UINT32                     Lbads;
  72   UINT32                     Flbas;
  73   UINT32                     LbaFmtIdx;
  74   UINT8                      Sn[21];
  75   UINT8                      Mn[41];
  76   VOID                       *DummyInterface;
  77
  78   NewDevicePathNode = NULL;
  79   DevicePath        = NULL;
  80   Device            = NULL;
  81
  82   //
  83   // Allocate a buffer for Identify Namespace data
  84   //
  85   NamespaceData = AllocateZeroPool (sizeof (NVME_ADMIN_NAMESPACE_DATA));
  86   if (NamespaceData == NULL) {
  87     return EFI_OUT_OF_RESOURCES;
  88   }
  89
  90   ParentDevicePath = Private->ParentDevicePath;
  91   //
  92   // Identify Namespace
  93   //
  94   Status = NvmeIdentifyNamespace (
  95              Private,
  96              NamespaceId,
  97              (VOID *)NamespaceData
  98              );
  99   if (EFI_ERROR (Status)) {
 100     goto Exit;
 101   }
 102
 103   //
 104   // Validate Namespace
 105   //
 106   if (NamespaceData->Ncap == 0) {
 107     Status = EFI_DEVICE_ERROR;
 108   } else {
 109     //
 110     // allocate device private data for each discovered namespace
 111     //
 112     Device = AllocateZeroPool (sizeof (NVME_DEVICE_PRIVATE_DATA));
 113     if (Device == NULL) {
 114       Status = EFI_OUT_OF_RESOURCES;
 115       goto Exit;
 116     }
 117
 118     //
 119     // Initialize SSD namespace instance data
 120     //
 121     Device->Signature     = NVME_DEVICE_PRIVATE_DATA_SIGNATURE;
 122     Device->NamespaceId   = NamespaceId;
 123     Device->NamespaceUuid = NamespaceData->Eui64;
 124
 125     Device->ControllerHandle    = Private->ControllerHandle;
 126     Device->DriverBindingHandle = Private->DriverBindingHandle;
 127     Device->Controller          = Private;
 128
 129     //
 130     // Build BlockIo media structure
 131     //
 132     Device->Media.MediaId          = 0;
 133     Device->Media.RemovableMedia   = FALSE;
 134     Device->Media.MediaPresent     = TRUE;
 135     Device->Media.LogicalPartition = FALSE;
 136     Device->Media.ReadOnly         = FALSE;
 137     Device->Media.WriteCaching     = FALSE;
 138     Device->Media.IoAlign          = Private->PassThruMode.IoAlign;
 139
 140     Flbas     = NamespaceData->Flbas;
 141     LbaFmtIdx = Flbas & 0xF;
 142
 143     //
 144     // Currently this NVME driver only suport Metadata Size == 0
 145     //
 146     if (NamespaceData->LbaFormat[LbaFmtIdx].Ms != 0) {
 147       DEBUG ((
 148         DEBUG_ERROR,
 149         "NVME IDENTIFY NAMESPACE [%d] Ms(%d) is not supported.\n",
 150         NamespaceId,
 151         NamespaceData->LbaFormat[LbaFmtIdx].Ms
 152         ));
 153       Status = EFI_UNSUPPORTED;
 154       goto Exit;
 155     }
 156
 157     Lbads                   = NamespaceData->LbaFormat[LbaFmtIdx].Lbads;
 158     Device->Media.BlockSize = (UINT32)1 << Lbads;
 159
 160     Device->Media.LastBlock                     = NamespaceData->Nsze - 1;
 161     Device->Media.LogicalBlocksPerPhysicalBlock = 1;
 162     Device->Media.LowestAlignedLba              = 1;
 163
 164     //
 165     // Create BlockIo Protocol instance
 166     //
 167     Device->BlockIo.Revision    = EFI_BLOCK_IO_PROTOCOL_REVISION2;
 168     Device->BlockIo.Media       = &Device->Media;
 169     Device->BlockIo.Reset       = NvmeBlockIoReset;
 170     Device->BlockIo.ReadBlocks  = NvmeBlockIoReadBlocks;
 171     Device->BlockIo.WriteBlocks = NvmeBlockIoWriteBlocks;
 172     Device->BlockIo.FlushBlocks = NvmeBlockIoFlushBlocks;
 173
 174     //
 175     // Create BlockIo2 Protocol instance
 176     //
 177     Device->BlockIo2.Media         = &Device->Media;
 178     Device->BlockIo2.Reset         = NvmeBlockIoResetEx;
 179     Device->BlockIo2.ReadBlocksEx  = NvmeBlockIoReadBlocksEx;
 180     Device->BlockIo2.WriteBlocksEx = NvmeBlockIoWriteBlocksEx;
 181     Device->BlockIo2.FlushBlocksEx = NvmeBlockIoFlushBlocksEx;
 182     InitializeListHead (&Device->AsyncQueue);
 183
 184     //
 185     // Create StorageSecurityProtocol Instance
 186     //
 187     Device->StorageSecurity.ReceiveData = NvmeStorageSecurityReceiveData;
 188     Device->StorageSecurity.SendData    = NvmeStorageSecuritySendData;
 189
 190     //
 191     // Create DiskInfo Protocol instance
 192     //
 193     CopyMem (&Device->NamespaceData, NamespaceData, sizeof (NVME_ADMIN_NAMESPACE_DATA));
 194     InitializeDiskInfo (Device);
 195
 196     //
 197     // Create a Nvm Express Namespace Device Path Node
 198     //
 199     Status = Private->Passthru.BuildDevicePath (
 200                                  &Private->Passthru,
 201                                  Device->NamespaceId,
 202                                  &NewDevicePathNode
 203                                  );
 204
 205     if (EFI_ERROR (Status)) {
 206       goto Exit;
 207     }
 208
 209     //
 210     // Append the SSD node to the controller's device path
 211     //
 212     DevicePath = AppendDevicePathNode (ParentDevicePath, NewDevicePathNode);
 213     if (DevicePath == NULL) {
 214       Status = EFI_OUT_OF_RESOURCES;
 215       goto Exit;
 216     }
 217
 218     DeviceHandle        = NULL;
 219     RemainingDevicePath = DevicePath;
 220     Status              = gBS->LocateDevicePath (&gEfiDevicePathProtocolGuid, &RemainingDevicePath, &DeviceHandle);
 221     if (!EFI_ERROR (Status) && (DeviceHandle != NULL) && IsDevicePathEnd (RemainingDevicePath)) {
 222       Status = EFI_ALREADY_STARTED;
 223       FreePool (DevicePath);
 224       goto Exit;
 225     }
 226
 227     Device->DevicePath = DevicePath;
 228
 229     //
 230     // Make sure the handle is NULL so we create a new handle
 231     //
 232     Device->DeviceHandle = NULL;
 233
 234     Status = gBS->InstallMultipleProtocolInterfaces (
 235                     &Device->DeviceHandle,
 236                     &gEfiDevicePathProtocolGuid,
 237                     Device->DevicePath,
 238                     &gEfiBlockIoProtocolGuid,
 239                     &Device->BlockIo,
 240                     &gEfiBlockIo2ProtocolGuid,
 241                     &Device->BlockIo2,
 242                     &gEfiDiskInfoProtocolGuid,
 243                     &Device->DiskInfo,
 244                     NULL
 245                     );
 246
 247     if (EFI_ERROR (Status)) {
 248       goto Exit;
 249     }
 250
 251     //
 252     // Check if the NVMe controller supports the Security Send and Security Receive commands
 253     //
 254     if ((Private->ControllerData->Oacs & SECURITY_SEND_RECEIVE_SUPPORTED) != 0) {
 255       Status = gBS->InstallProtocolInterface (
 256                       &Device->DeviceHandle,
 257                       &gEfiStorageSecurityCommandProtocolGuid,
 258                       EFI_NATIVE_INTERFACE,
 259                       &Device->StorageSecurity
 260                       );
 261       if (EFI_ERROR (Status)) {
 262         gBS->UninstallMultipleProtocolInterfaces (
 263                Device->DeviceHandle,
 264                &gEfiDevicePathProtocolGuid,
 265                Device->DevicePath,
 266                &gEfiBlockIoProtocolGuid,
 267                &Device->BlockIo,
 268                &gEfiBlockIo2ProtocolGuid,
 269                &Device->BlockIo2,
 270                &gEfiDiskInfoProtocolGuid,
 271                &Device->DiskInfo,
 272                NULL
 273                );
 274         goto Exit;
 275       }
 276     }
 277
 278     gBS->OpenProtocol (
 279            Private->ControllerHandle,
 280            &gEfiNvmExpressPassThruProtocolGuid,
 281            (VOID **)&DummyInterface,
 282            Private->DriverBindingHandle,
 283            Device->DeviceHandle,
 284            EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER
 285            );
 286
 287     //
 288     // Dump NvmExpress Identify Namespace Data
 289     //
 290     DEBUG ((DEBUG_INFO, " == NVME IDENTIFY NAMESPACE [%d] DATA ==\n", NamespaceId));
 291     DEBUG ((DEBUG_INFO, "    NSZE        : 0x%x\n", NamespaceData->Nsze));
 292     DEBUG ((DEBUG_INFO, "    NCAP        : 0x%x\n", NamespaceData->Ncap));
 293     DEBUG ((DEBUG_INFO, "    NUSE        : 0x%x\n", NamespaceData->Nuse));
 294     DEBUG ((DEBUG_INFO, "    LBAF0.LBADS : 0x%x\n", (NamespaceData->LbaFormat[0].Lbads)));
 295
 296     //
 297     // Build controller name for Component Name (2) protocol.
 298     //
 299     CopyMem (Sn, Private->ControllerData->Sn, sizeof (Private->ControllerData->Sn));
 300     Sn[20] = 0;
 301     CopyMem (Mn, Private->ControllerData->Mn, sizeof (Private->ControllerData->Mn));
 302     Mn[40] = 0;
 303     UnicodeSPrintAsciiFormat (Device->ModelName, sizeof (Device->ModelName), "%a-%a-%x", Sn, Mn, NamespaceData->Eui64);
 304
 305     AddUnicodeString2 (
 306       "eng",
 307       gNvmExpressComponentName.SupportedLanguages,
 308       &Device->ControllerNameTable,
 309       Device->ModelName,
 310       TRUE
 311       );
 312
 313     AddUnicodeString2 (
 314       "en",
 315       gNvmExpressComponentName2.SupportedLanguages,
 316       &Device->ControllerNameTable,
 317       Device->ModelName,
 318       FALSE
 319       );
 320   }
 321
 322 Exit:
 323   if (NamespaceData != NULL) {
 324     FreePool (NamespaceData);
 325   }
 326
 327   if (NewDevicePathNode != NULL) {
 328     FreePool (NewDevicePathNode);
 329   }
 330
 331   if (EFI_ERROR (Status) && (Device != NULL) && (Device->DevicePath != NULL)) {
 332     FreePool (Device->DevicePath);
 333   }
 334
 335   if (EFI_ERROR (Status) && (Device != NULL)) {
 336     FreePool (Device);
 337   }
 338
 339   return Status;
 340 }
 341
 342 /**
 343   Discover all Nvm Express device namespaces, and create child handles for them with BlockIo
 344   and DiskInfo protocol instances.
 345
 346   @param[in] Private         The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 347
 348   @retval EFI_SUCCESS        All the namespaces in the device are successfully enumerated.
 349   @return Others             Some error occurs when enumerating the namespaces.
 350
 351 **/
 352 EFI_STATUS
 353 DiscoverAllNamespaces (
 354   IN NVME_CONTROLLER_PRIVATE_DATA  *Private
 355   )
 356 {
 357   EFI_STATUS                          Status;
 358   UINT32                              NamespaceId;
 359   EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL  *Passthru;
 360
 361   NamespaceId = 0xFFFFFFFF;
 362   Passthru    = &Private->Passthru;
 363
 364   while (TRUE) {
 365     Status = Passthru->GetNextNamespace (
 366                          Passthru,
 367                          (UINT32 *)&NamespaceId
 368                          );
 369
 370     if (EFI_ERROR (Status)) {
 371       break;
 372     }
 373
 374     Status = EnumerateNvmeDevNamespace (
 375                Private,
 376                NamespaceId
 377                );
 378
 379     if (EFI_ERROR (Status)) {
 380       continue;
 381     }
 382   }
 383
 384   return EFI_SUCCESS;
 385 }
 386
 387 /**
 388   Unregisters a Nvm Express device namespace.
 389
 390   This function removes the protocols installed on the controller handle and
 391   frees the resources allocated for the namespace.
 392
 393   @param  This                  The pointer to EFI_DRIVER_BINDING_PROTOCOL instance.
 394   @param  Controller            The controller handle of the namespace.
 395   @param  Handle                The child handle.
 396
 397   @retval EFI_SUCCESS           The namespace is successfully unregistered.
 398   @return Others                Some error occurs when unregistering the namespace.
 399
 400 **/
 401 EFI_STATUS
 402 UnregisterNvmeNamespace (
 403   IN  EFI_DRIVER_BINDING_PROTOCOL  *This,
 404   IN  EFI_HANDLE                   Controller,
 405   IN  EFI_HANDLE                   Handle
 406   )
 407 {
 408   EFI_STATUS                             Status;
 409   EFI_BLOCK_IO_PROTOCOL                  *BlockIo;
 410   NVME_DEVICE_PRIVATE_DATA               *Device;
 411   EFI_STORAGE_SECURITY_COMMAND_PROTOCOL  *StorageSecurity;
 412   BOOLEAN                                IsEmpty;
 413   EFI_TPL                                OldTpl;
 414   VOID                                   *DummyInterface;
 415
 416   BlockIo = NULL;
 417
 418   Status = gBS->OpenProtocol (
 419                   Handle,
 420                   &gEfiBlockIoProtocolGuid,
 421                   (VOID **)&BlockIo,
 422                   This->DriverBindingHandle,
 423                   Controller,
 424                   EFI_OPEN_PROTOCOL_GET_PROTOCOL
 425                   );
 426   if (EFI_ERROR (Status)) {
 427     return Status;
 428   }
 429
 430   Device = NVME_DEVICE_PRIVATE_DATA_FROM_BLOCK_IO (BlockIo);
 431
 432   //
 433   // Wait for the device's asynchronous I/O queue to become empty.
 434   //
 435   while (TRUE) {
 436     OldTpl  = gBS->RaiseTPL (TPL_NOTIFY);
 437     IsEmpty = IsListEmpty (&Device->AsyncQueue);
 438     gBS->RestoreTPL (OldTpl);
 439
 440     if (IsEmpty) {
 441       break;
 442     }
 443
 444     gBS->Stall (100);
 445   }
 446
 447   //
 448   // Close the child handle
 449   //
 450   gBS->CloseProtocol (
 451          Controller,
 452          &gEfiNvmExpressPassThruProtocolGuid,
 453          This->DriverBindingHandle,
 454          Handle
 455          );
 456
 457   //
 458   // The Nvm Express driver installs the BlockIo and DiskInfo in the DriverBindingStart().
 459   // Here should uninstall both of them.
 460   //
 461   Status = gBS->UninstallMultipleProtocolInterfaces (
 462                   Handle,
 463                   &gEfiDevicePathProtocolGuid,
 464                   Device->DevicePath,
 465                   &gEfiBlockIoProtocolGuid,
 466                   &Device->BlockIo,
 467                   &gEfiBlockIo2ProtocolGuid,
 468                   &Device->BlockIo2,
 469                   &gEfiDiskInfoProtocolGuid,
 470                   &Device->DiskInfo,
 471                   NULL
 472                   );
 473
 474   if (EFI_ERROR (Status)) {
 475     gBS->OpenProtocol (
 476            Controller,
 477            &gEfiNvmExpressPassThruProtocolGuid,
 478            (VOID **)&DummyInterface,
 479            This->DriverBindingHandle,
 480            Handle,
 481            EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER
 482            );
 483     return Status;
 484   }
 485
 486   //
 487   // If Storage Security Command Protocol is installed, then uninstall this protocol.
 488   //
 489   Status = gBS->OpenProtocol (
 490                   Handle,
 491                   &gEfiStorageSecurityCommandProtocolGuid,
 492                   (VOID **)&StorageSecurity,
 493                   This->DriverBindingHandle,
 494                   Controller,
 495                   EFI_OPEN_PROTOCOL_GET_PROTOCOL
 496                   );
 497
 498   if (!EFI_ERROR (Status)) {
 499     Status = gBS->UninstallProtocolInterface (
 500                     Handle,
 501                     &gEfiStorageSecurityCommandProtocolGuid,
 502                     &Device->StorageSecurity
 503                     );
 504     if (EFI_ERROR (Status)) {
 505       gBS->OpenProtocol (
 506              Controller,
 507              &gEfiNvmExpressPassThruProtocolGuid,
 508              (VOID **)&DummyInterface,
 509              This->DriverBindingHandle,
 510              Handle,
 511              EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER
 512              );
 513       return Status;
 514     }
 515   }
 516
 517   if (Device->DevicePath != NULL) {
 518     FreePool (Device->DevicePath);
 519   }
 520
 521   if (Device->ControllerNameTable != NULL) {
 522     FreeUnicodeStringTable (Device->ControllerNameTable);
 523   }
 524
 525   FreePool (Device);
 526
 527   return EFI_SUCCESS;
 528 }
 529
 530 /**
 531   Call back function when the timer event is signaled.
 532
 533   @param[in]  Event     The Event this notify function registered to.
 534   @param[in]  Context   Pointer to the context data registered to the
 535                         Event.
 536
 537 **/
 538 VOID
 539 EFIAPI
 540 ProcessAsyncTaskList (
 541   IN EFI_EVENT  Event,
 542   IN VOID       *Context
 543   )
 544 {
 545   NVME_CONTROLLER_PRIVATE_DATA  *Private;
 546   EFI_PCI_IO_PROTOCOL           *PciIo;
 547   NVME_CQ                       *Cq;
 548   UINT16                        QueueId;
 549   UINT32                        Data;
 550   LIST_ENTRY                    *Link;
 551   LIST_ENTRY                    *NextLink;
 552   NVME_PASS_THRU_ASYNC_REQ      *AsyncRequest;
 553   NVME_BLKIO2_SUBTASK           *Subtask;
 554   NVME_BLKIO2_REQUEST           *BlkIo2Request;
 555   EFI_BLOCK_IO2_TOKEN           *Token;
 556   BOOLEAN                       HasNewItem;
 557   EFI_STATUS                    Status;
 558
 559   Private    = (NVME_CONTROLLER_PRIVATE_DATA *)Context;
 560   QueueId    = 2;
 561   Cq         = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
 562   HasNewItem = FALSE;
 563   PciIo      = Private->PciIo;
 564
 565   //
 566   // Submit asynchronous subtasks to the NVMe Submission Queue
 567   //
 568   for (Link = GetFirstNode (&Private->UnsubmittedSubtasks);
 569        !IsNull (&Private->UnsubmittedSubtasks, Link);
 570        Link = NextLink)
 571   {
 572     NextLink      = GetNextNode (&Private->UnsubmittedSubtasks, Link);
 573     Subtask       = NVME_BLKIO2_SUBTASK_FROM_LINK (Link);
 574     BlkIo2Request = Subtask->BlockIo2Request;
 575     Token         = BlkIo2Request->Token;
 576     RemoveEntryList (Link);
 577     BlkIo2Request->UnsubmittedSubtaskNum--;
 578
 579     //
 580     // If any previous subtask fails, do not process subsequent ones.
 581     //
 582     if (Token->TransactionStatus != EFI_SUCCESS) {
 583       if (IsListEmpty (&BlkIo2Request->SubtasksQueue) &&
 584           BlkIo2Request->LastSubtaskSubmitted &&
 585           (BlkIo2Request->UnsubmittedSubtaskNum == 0))
 586       {
 587         //
 588         // Remove the BlockIo2 request from the device asynchronous queue.
 589         //
 590         RemoveEntryList (&BlkIo2Request->Link);
 591         FreePool (BlkIo2Request);
 592         gBS->SignalEvent (Token->Event);
 593       }
 594
 595       FreePool (Subtask->CommandPacket->NvmeCmd);
 596       FreePool (Subtask->CommandPacket->NvmeCompletion);
 597       FreePool (Subtask->CommandPacket);
 598       FreePool (Subtask);
 599
 600       continue;
 601     }
 602
 603     Status = Private->Passthru.PassThru (
 604                                  &Private->Passthru,
 605                                  Subtask->NamespaceId,
 606                                  Subtask->CommandPacket,
 607                                  Subtask->Event
 608                                  );
 609     if (Status == EFI_NOT_READY) {
 610       InsertHeadList (&Private->UnsubmittedSubtasks, Link);
 611       BlkIo2Request->UnsubmittedSubtaskNum++;
 612       break;
 613     } else if (EFI_ERROR (Status)) {
 614       Token->TransactionStatus = EFI_DEVICE_ERROR;
 615
 616       if (IsListEmpty (&BlkIo2Request->SubtasksQueue) &&
 617           Subtask->IsLast)
 618       {
 619         //
 620         // Remove the BlockIo2 request from the device asynchronous queue.
 621         //
 622         RemoveEntryList (&BlkIo2Request->Link);
 623         FreePool (BlkIo2Request);
 624         gBS->SignalEvent (Token->Event);
 625       }
 626
 627       FreePool (Subtask->CommandPacket->NvmeCmd);
 628       FreePool (Subtask->CommandPacket->NvmeCompletion);
 629       FreePool (Subtask->CommandPacket);
 630       FreePool (Subtask);
 631     } else {
 632       InsertTailList (&BlkIo2Request->SubtasksQueue, Link);
 633       if (Subtask->IsLast) {
 634         BlkIo2Request->LastSubtaskSubmitted = TRUE;
 635       }
 636     }
 637   }
 638
 639   while (Cq->Pt != Private->Pt[QueueId]) {
 640     ASSERT (Cq->Sqid == QueueId);
 641
 642     HasNewItem = TRUE;
 643
 644     //
 645     // Find the command with given Command Id.
 646     //
 647     for (Link = GetFirstNode (&Private->AsyncPassThruQueue);
 648          !IsNull (&Private->AsyncPassThruQueue, Link);
 649          Link = NextLink)
 650     {
 651       NextLink     = GetNextNode (&Private->AsyncPassThruQueue, Link);
 652       AsyncRequest = NVME_PASS_THRU_ASYNC_REQ_FROM_THIS (Link);
 653       if (AsyncRequest->CommandId == Cq->Cid) {
 654         //
 655         // Copy the Respose Queue entry for this command to the callers
 656         // response buffer.
 657         //
 658         CopyMem (
 659           AsyncRequest->Packet->NvmeCompletion,
 660           Cq,
 661           sizeof (EFI_NVM_EXPRESS_COMPLETION)
 662           );
 663
 664         //
 665         // Free the resources allocated before cmd submission
 666         //
 667         if (AsyncRequest->MapData != NULL) {
 668           PciIo->Unmap (PciIo, AsyncRequest->MapData);
 669         }
 670
 671         if (AsyncRequest->MapMeta != NULL) {
 672           PciIo->Unmap (PciIo, AsyncRequest->MapMeta);
 673         }
 674
 675         if (AsyncRequest->MapPrpList != NULL) {
 676           PciIo->Unmap (PciIo, AsyncRequest->MapPrpList);
 677         }
 678
 679         if (AsyncRequest->PrpListHost != NULL) {
 680           PciIo->FreeBuffer (
 681                    PciIo,
 682                    AsyncRequest->PrpListNo,
 683                    AsyncRequest->PrpListHost
 684                    );
 685         }
 686
 687         RemoveEntryList (Link);
 688         gBS->SignalEvent (AsyncRequest->CallerEvent);
 689         FreePool (AsyncRequest);
 690
 691         //
 692         // Update submission queue head.
 693         //
 694         Private->AsyncSqHead = Cq->Sqhd;
 695         break;
 696       }
 697     }
 698
 699     Private->CqHdbl[QueueId].Cqh++;
 700     if (Private->CqHdbl[QueueId].Cqh > MIN (NVME_ASYNC_CCQ_SIZE, Private->Cap.Mqes)) {
 701       Private->CqHdbl[QueueId].Cqh = 0;
 702       Private->Pt[QueueId]        ^= 1;
 703     }
 704
 705     Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
 706   }
 707
 708   if (HasNewItem) {
 709     Data = ReadUnaligned32 ((UINT32 *)&Private->CqHdbl[QueueId]);
 710     PciIo->Mem.Write (
 711                  PciIo,
 712                  EfiPciIoWidthUint32,
 713                  NVME_BAR,
 714                  NVME_CQHDBL_OFFSET (QueueId, Private->Cap.Dstrd),
 715                  1,
 716                  &Data
 717                  );
 718   }
 719 }
 720
 721 /**
 722   Tests to see if this driver supports a given controller. If a child device is provided,
 723   it further tests to see if this driver supports creating a handle for the specified child device.
 724
 725   This function checks to see if the driver specified by This supports the device specified by
 726   ControllerHandle. Drivers will typically use the device path attached to
 727   ControllerHandle and/or the services from the bus I/O abstraction attached to
 728   ControllerHandle to determine if the driver supports ControllerHandle. This function
 729   may be called many times during platform initialization. In order to reduce boot times, the tests
 730   performed by this function must be very small, and take as little time as possible to execute. This
 731   function must not change the state of any hardware devices, and this function must be aware that the
 732   device specified by ControllerHandle may already be managed by the same driver or a
 733   different driver. This function must match its calls to AllocatePages() with FreePages(),
 734   AllocatePool() with FreePool(), and OpenProtocol() with CloseProtocol().
 735   Since ControllerHandle may have been previously started by the same driver, if a protocol is
 736   already in the opened state, then it must not be closed with CloseProtocol(). This is required
 737   to guarantee the state of ControllerHandle is not modified by this function.
 738
 739   @param[in]  This                 A pointer to the EFI_DRIVER_BINDING_PROTOCOL instance.
 740   @param[in]  ControllerHandle     The handle of the controller to test. This handle
 741                                    must support a protocol interface that supplies
 742                                    an I/O abstraction to the driver.
 743   @param[in]  RemainingDevicePath  A pointer to the remaining portion of a device path.  This
 744                                    parameter is ignored by device drivers, and is optional for bus
 745                                    drivers. For bus drivers, if this parameter is not NULL, then
 746                                    the bus driver must determine if the bus controller specified
 747                                    by ControllerHandle and the child controller specified
 748                                    by RemainingDevicePath are both supported by this
 749                                    bus driver.
 750
 751   @retval EFI_SUCCESS              The device specified by ControllerHandle and
 752                                    RemainingDevicePath is supported by the driver specified by This.
 753   @retval EFI_ALREADY_STARTED      The device specified by ControllerHandle and
 754                                    RemainingDevicePath is already being managed by the driver
 755                                    specified by This.
 756   @retval EFI_ACCESS_DENIED        The device specified by ControllerHandle and
 757                                    RemainingDevicePath is already being managed by a different
 758                                    driver or an application that requires exclusive access.
 759                                    Currently not implemented.
 760   @retval EFI_UNSUPPORTED          The device specified by ControllerHandle and
 761                                    RemainingDevicePath is not supported by the driver specified by This.
 762 **/
 763 EFI_STATUS
 764 EFIAPI
 765 NvmExpressDriverBindingSupported (
 766   IN EFI_DRIVER_BINDING_PROTOCOL  *This,
 767   IN EFI_HANDLE                   Controller,
 768   IN EFI_DEVICE_PATH_PROTOCOL     *RemainingDevicePath
 769   )
 770 {
 771   EFI_STATUS                Status;
 772   EFI_DEV_PATH_PTR          DevicePathNode;
 773   EFI_DEVICE_PATH_PROTOCOL  *ParentDevicePath;
 774   EFI_PCI_IO_PROTOCOL       *PciIo;
 775   UINT8                     ClassCode[3];
 776
 777   //
 778   // Check whether device path is valid
 779   //
 780   if (RemainingDevicePath != NULL) {
 781     //
 782     // Check if RemainingDevicePath is the End of Device Path Node,
 783     // if yes, go on checking other conditions
 784     //
 785     if (!IsDevicePathEnd (RemainingDevicePath)) {
 786       //
 787       // If RemainingDevicePath isn't the End of Device Path Node,
 788       // check its validation
 789       //
 790       DevicePathNode.DevPath = RemainingDevicePath;
 791
 792       if ((DevicePathNode.DevPath->Type    != MESSAGING_DEVICE_PATH) ||
 793           (DevicePathNode.DevPath->SubType != MSG_NVME_NAMESPACE_DP) ||
 794           (DevicePathNodeLength (DevicePathNode.DevPath) != sizeof (NVME_NAMESPACE_DEVICE_PATH)))
 795       {
 796         return EFI_UNSUPPORTED;
 797       }
 798     }
 799   }
 800
 801   //
 802   // Open the EFI Device Path protocol needed to perform the supported test
 803   //
 804   Status = gBS->OpenProtocol (
 805                   Controller,
 806                   &gEfiDevicePathProtocolGuid,
 807                   (VOID **)&ParentDevicePath,
 808                   This->DriverBindingHandle,
 809                   Controller,
 810                   EFI_OPEN_PROTOCOL_BY_DRIVER
 811                   );
 812   if (Status == EFI_ALREADY_STARTED) {
 813     return EFI_SUCCESS;
 814   }
 815
 816   if (EFI_ERROR (Status)) {
 817     return Status;
 818   }
 819
 820   //
 821   // Close protocol, don't use device path protocol in the Support() function
 822   //
 823   gBS->CloseProtocol (
 824          Controller,
 825          &gEfiDevicePathProtocolGuid,
 826          This->DriverBindingHandle,
 827          Controller
 828          );
 829
 830   //
 831   // Attempt to Open PCI I/O Protocol
 832   //
 833   Status = gBS->OpenProtocol (
 834                   Controller,
 835                   &gEfiPciIoProtocolGuid,
 836                   (VOID **)&PciIo,
 837                   This->DriverBindingHandle,
 838                   Controller,
 839                   EFI_OPEN_PROTOCOL_BY_DRIVER
 840                   );
 841   if (Status == EFI_ALREADY_STARTED) {
 842     return EFI_SUCCESS;
 843   }
 844
 845   if (EFI_ERROR (Status)) {
 846     return Status;
 847   }
 848
 849   //
 850   // Now further check the PCI header: Base class (offset 0x0B) and Sub Class (offset 0x0A).
 851   // This controller should be a Nvm Express controller.
 852   //
 853   Status = PciIo->Pci.Read (
 854                         PciIo,
 855                         EfiPciIoWidthUint8,
 856                         PCI_CLASSCODE_OFFSET,
 857                         sizeof (ClassCode),
 858                         ClassCode
 859                         );
 860   if (EFI_ERROR (Status)) {
 861     goto Done;
 862   }
 863
 864   //
 865   // Examine Nvm Express controller PCI Configuration table fields
 866   //
 867   if ((ClassCode[0] != PCI_IF_NVMHCI) || (ClassCode[1] != PCI_CLASS_MASS_STORAGE_NVM) || (ClassCode[2] != PCI_CLASS_MASS_STORAGE)) {
 868     Status = EFI_UNSUPPORTED;
 869   }
 870
 871 Done:
 872   gBS->CloseProtocol (
 873          Controller,
 874          &gEfiPciIoProtocolGuid,
 875          This->DriverBindingHandle,
 876          Controller
 877          );
 878
 879   return Status;
 880 }
 881
 882 /**
 883   Starts a device controller or a bus controller.
 884
 885   The Start() function is designed to be invoked from the EFI boot service ConnectController().
 886   As a result, much of the error checking on the parameters to Start() has been moved into this
 887   common boot service. It is legal to call Start() from other locations,
 888   but the following calling restrictions must be followed or the system behavior will not be deterministic.
 889   1. ControllerHandle must be a valid EFI_HANDLE.
 890   2. If RemainingDevicePath is not NULL, then it must be a pointer to a naturally aligned
 891      EFI_DEVICE_PATH_PROTOCOL.
 892   3. Prior to calling Start(), the Supported() function for the driver specified by This must
 893      have been called with the same calling parameters, and Supported() must have returned EFI_SUCCESS.
 894
 895   @param[in]  This                 A pointer to the EFI_DRIVER_BINDING_PROTOCOL instance.
 896   @param[in]  ControllerHandle     The handle of the controller to start. This handle
 897                                    must support a protocol interface that supplies
 898                                    an I/O abstraction to the driver.
 899   @param[in]  RemainingDevicePath  A pointer to the remaining portion of a device path.  This
 900                                    parameter is ignored by device drivers, and is optional for bus
 901                                    drivers. For a bus driver, if this parameter is NULL, then handles
 902                                    for all the children of Controller are created by this driver.
 903                                    If this parameter is not NULL and the first Device Path Node is
 904                                    not the End of Device Path Node, then only the handle for the
 905                                    child device specified by the first Device Path Node of
 906                                    RemainingDevicePath is created by this driver.
 907                                    If the first Device Path Node of RemainingDevicePath is
 908                                    the End of Device Path Node, no child handle is created by this
 909                                    driver.
 910
 911   @retval EFI_SUCCESS              The device was started.
 912   @retval EFI_DEVICE_ERROR         The device could not be started due to a device error.Currently not implemented.
 913   @retval EFI_OUT_OF_RESOURCES     The request could not be completed due to a lack of resources.
 914   @retval Others                   The driver failded to start the device.
 915
 916 **/
 917 EFI_STATUS
 918 EFIAPI
 919 NvmExpressDriverBindingStart (
 920   IN EFI_DRIVER_BINDING_PROTOCOL  *This,
 921   IN EFI_HANDLE                   Controller,
 922   IN EFI_DEVICE_PATH_PROTOCOL     *RemainingDevicePath
 923   )
 924 {
 925   EFI_STATUS                          Status;
 926   EFI_PCI_IO_PROTOCOL                 *PciIo;
 927   NVME_CONTROLLER_PRIVATE_DATA        *Private;
 928   EFI_DEVICE_PATH_PROTOCOL            *ParentDevicePath;
 929   UINT32                              NamespaceId;
 930   EFI_PHYSICAL_ADDRESS                MappedAddr;
 931   UINTN                               Bytes;
 932   EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL  *Passthru;
 933
 934   DEBUG ((DEBUG_INFO, "NvmExpressDriverBindingStart: start\n"));
 935
 936   Private          = NULL;
 937   Passthru         = NULL;
 938   ParentDevicePath = NULL;
 939
 940   Status = gBS->OpenProtocol (
 941                   Controller,
 942                   &gEfiDevicePathProtocolGuid,
 943                   (VOID **)&ParentDevicePath,
 944                   This->DriverBindingHandle,
 945                   Controller,
 946                   EFI_OPEN_PROTOCOL_BY_DRIVER
 947                   );
 948   if ((EFI_ERROR (Status)) && (Status != EFI_ALREADY_STARTED)) {
 949     return Status;
 950   }
 951
 952   Status = gBS->OpenProtocol (
 953                   Controller,
 954                   &gEfiPciIoProtocolGuid,
 955                   (VOID **)&PciIo,
 956                   This->DriverBindingHandle,
 957                   Controller,
 958                   EFI_OPEN_PROTOCOL_BY_DRIVER
 959                   );
 960
 961   if (EFI_ERROR (Status) && (Status != EFI_ALREADY_STARTED)) {
 962     return Status;
 963   }
 964
 965   //
 966   // Check EFI_ALREADY_STARTED to reuse the original NVME_CONTROLLER_PRIVATE_DATA.
 967   //
 968   if (Status != EFI_ALREADY_STARTED) {
 969     Private = AllocateZeroPool (sizeof (NVME_CONTROLLER_PRIVATE_DATA));
 970
 971     if (Private == NULL) {
 972       DEBUG ((DEBUG_ERROR, "NvmExpressDriverBindingStart: allocating pool for Nvme Private Data failed!\n"));
 973       Status = EFI_OUT_OF_RESOURCES;
 974       goto Exit;
 975     }
 976
 977     //
 978     // Save original PCI attributes
 979     //
 980     Status = PciIo->Attributes (
 981                       PciIo,
 982                       EfiPciIoAttributeOperationGet,
 983                       0,
 984                       &Private->PciAttributes
 985                       );
 986
 987     if (EFI_ERROR (Status)) {
 988       return Status;
 989     }
 990
 991     //
 992     // Enable 64-bit DMA support in the PCI layer.
 993     //
 994     Status = PciIo->Attributes (
 995                       PciIo,
 996                       EfiPciIoAttributeOperationEnable,
 997                       EFI_PCI_IO_ATTRIBUTE_DUAL_ADDRESS_CYCLE,
 998                       NULL
 999                       );
1000     if (EFI_ERROR (Status)) {
1001       DEBUG ((DEBUG_WARN, "NvmExpressDriverBindingStart: failed to enable 64-bit DMA (%r)\n", Status));
1002     }
1003
1004     //
1005     // 6 x 4kB aligned buffers will be carved out of this buffer.
1006     // 1st 4kB boundary is the start of the admin submission queue.
1007     // 2nd 4kB boundary is the start of the admin completion queue.
1008     // 3rd 4kB boundary is the start of I/O submission queue #1.
1009     // 4th 4kB boundary is the start of I/O completion queue #1.
1010     // 5th 4kB boundary is the start of I/O submission queue #2.
1011     // 6th 4kB boundary is the start of I/O completion queue #2.
1012     //
1013     // Allocate 6 pages of memory, then map it for bus master read and write.
1014     //
1015     Status = PciIo->AllocateBuffer (
1016                       PciIo,
1017                       AllocateAnyPages,
1018                       EfiBootServicesData,
1019                       6,
1020                       (VOID **)&Private->Buffer,
1021                       0
1022                       );
1023     if (EFI_ERROR (Status)) {
1024       goto Exit;
1025     }
1026
1027     Bytes  = EFI_PAGES_TO_SIZE (6);
1028     Status = PciIo->Map (
1029                       PciIo,
1030                       EfiPciIoOperationBusMasterCommonBuffer,
1031                       Private->Buffer,
1032                       &Bytes,
1033                       &MappedAddr,
1034                       &Private->Mapping
1035                       );
1036
1037     if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (6))) {
1038       goto Exit;
1039     }
1040
1041     Private->BufferPciAddr = (UINT8 *)(UINTN)MappedAddr;
1042
1043     Private->Signature                 = NVME_CONTROLLER_PRIVATE_DATA_SIGNATURE;
1044     Private->ControllerHandle          = Controller;
1045     Private->ImageHandle               = This->DriverBindingHandle;
1046     Private->DriverBindingHandle       = This->DriverBindingHandle;
1047     Private->PciIo                     = PciIo;
1048     Private->ParentDevicePath          = ParentDevicePath;
1049     Private->Passthru.Mode             = &Private->PassThruMode;
1050     Private->Passthru.PassThru         = NvmExpressPassThru;
1051     Private->Passthru.GetNextNamespace = NvmExpressGetNextNamespace;
1052     Private->Passthru.BuildDevicePath  = NvmExpressBuildDevicePath;
1053     Private->Passthru.GetNamespace     = NvmExpressGetNamespace;
1054     CopyMem (&Private->PassThruMode, &gEfiNvmExpressPassThruMode, sizeof (EFI_NVM_EXPRESS_PASS_THRU_MODE));
1055     InitializeListHead (&Private->AsyncPassThruQueue);
1056     InitializeListHead (&Private->UnsubmittedSubtasks);
1057
1058     Status = NvmeControllerInit (Private);
1059     if (EFI_ERROR (Status)) {
1060       goto Exit;
1061     }
1062
1063     //
1064     // Start the asynchronous I/O completion monitor
1065     //
1066     Status = gBS->CreateEvent (
1067                     EVT_TIMER | EVT_NOTIFY_SIGNAL,
1068                     TPL_NOTIFY,
1069                     ProcessAsyncTaskList,
1070                     Private,
1071                     &Private->TimerEvent
1072                     );
1073     if (EFI_ERROR (Status)) {
1074       goto Exit;
1075     }
1076
1077     Status = gBS->SetTimer (
1078                     Private->TimerEvent,
1079                     TimerPeriodic,
1080                     NVME_HC_ASYNC_TIMER
1081                     );
1082     if (EFI_ERROR (Status)) {
1083       goto Exit;
1084     }
1085
1086     Status = gBS->InstallMultipleProtocolInterfaces (
1087                     &Controller,
1088                     &gEfiNvmExpressPassThruProtocolGuid,
1089                     &Private->Passthru,
1090                     NULL
1091                     );
1092     if (EFI_ERROR (Status)) {
1093       goto Exit;
1094     }
1095
1096     NvmeRegisterShutdownNotification ();
1097   } else {
1098     Status = gBS->OpenProtocol (
1099                     Controller,
1100                     &gEfiNvmExpressPassThruProtocolGuid,
1101                     (VOID **)&Passthru,
1102                     This->DriverBindingHandle,
1103                     Controller,
1104                     EFI_OPEN_PROTOCOL_GET_PROTOCOL
1105                     );
1106     if (EFI_ERROR (Status)) {
1107       goto Exit;
1108     }
1109
1110     Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (Passthru);
1111   }
1112
1113   if (RemainingDevicePath == NULL) {
1114     //
1115     // Enumerate all NVME namespaces in the controller
1116     //
1117     Status = DiscoverAllNamespaces (
1118                Private
1119                );
1120   } else if (!IsDevicePathEnd (RemainingDevicePath)) {
1121     //
1122     // Enumerate the specified NVME namespace
1123     //
1124     Status = Private->Passthru.GetNamespace (
1125                                  &Private->Passthru,
1126                                  RemainingDevicePath,
1127                                  &NamespaceId
1128                                  );
1129
1130     if (!EFI_ERROR (Status)) {
1131       Status = EnumerateNvmeDevNamespace (
1132                  Private,
1133                  NamespaceId
1134                  );
1135     }
1136   }
1137
1138   DEBUG ((DEBUG_INFO, "NvmExpressDriverBindingStart: end successfully\n"));
1139   return EFI_SUCCESS;
1140
1141 Exit:
1142   if ((Private != NULL) && (Private->Mapping != NULL)) {
1143     PciIo->Unmap (PciIo, Private->Mapping);
1144   }
1145
1146   if ((Private != NULL) && (Private->Buffer != NULL)) {
1147     PciIo->FreeBuffer (PciIo, 6, Private->Buffer);
1148   }
1149
1150   if ((Private != NULL) && (Private->ControllerData != NULL)) {
1151     FreePool (Private->ControllerData);
1152   }
1153
1154   if (Private != NULL) {
1155     if (Private->TimerEvent != NULL) {
1156       gBS->CloseEvent (Private->TimerEvent);
1157     }
1158
1159     FreePool (Private);
1160   }
1161
1162   gBS->CloseProtocol (
1163          Controller,
1164          &gEfiPciIoProtocolGuid,
1165          This->DriverBindingHandle,
1166          Controller
1167          );
1168
1169   gBS->CloseProtocol (
1170          Controller,
1171          &gEfiDevicePathProtocolGuid,
1172          This->DriverBindingHandle,
1173          Controller
1174          );
1175
1176   DEBUG ((DEBUG_INFO, "NvmExpressDriverBindingStart: end with %r\n", Status));
1177
1178   return Status;
1179 }
1180
1181 /**
1182   Stops a device controller or a bus controller.
1183
1184   The Stop() function is designed to be invoked from the EFI boot service DisconnectController().
1185   As a result, much of the error checking on the parameters to Stop() has been moved
1186   into this common boot service. It is legal to call Stop() from other locations,
1187   but the following calling restrictions must be followed or the system behavior will not be deterministic.
1188   1. ControllerHandle must be a valid EFI_HANDLE that was used on a previous call to this
1189      same driver's Start() function.
1190   2. The first NumberOfChildren handles of ChildHandleBuffer must all be a valid
1191      EFI_HANDLE. In addition, all of these handles must have been created in this driver's
1192      Start() function, and the Start() function must have called OpenProtocol() on
1193      ControllerHandle with an Attribute of EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER.
1194
1195   @param[in]  This              A pointer to the EFI_DRIVER_BINDING_PROTOCOL instance.
1196   @param[in]  ControllerHandle  A handle to the device being stopped. The handle must
1197                                 support a bus specific I/O protocol for the driver
1198                                 to use to stop the device.
1199   @param[in]  NumberOfChildren  The number of child device handles in ChildHandleBuffer.
1200   @param[in]  ChildHandleBuffer An array of child handles to be freed. May be NULL
1201                                 if NumberOfChildren is 0.
1202
1203   @retval EFI_SUCCESS           The device was stopped.
1204   @retval EFI_DEVICE_ERROR      The device could not be stopped due to a device error.
1205
1206 **/
1207 EFI_STATUS
1208 EFIAPI
1209 NvmExpressDriverBindingStop (
1210   IN  EFI_DRIVER_BINDING_PROTOCOL  *This,
1211   IN  EFI_HANDLE                   Controller,
1212   IN  UINTN                        NumberOfChildren,
1213   IN  EFI_HANDLE                   *ChildHandleBuffer
1214   )
1215 {
1216   EFI_STATUS                          Status;
1217   BOOLEAN                             AllChildrenStopped;
1218   UINTN                               Index;
1219   NVME_CONTROLLER_PRIVATE_DATA        *Private;
1220   EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL  *PassThru;
1221   BOOLEAN                             IsEmpty;
1222   EFI_TPL                             OldTpl;
1223
1224   if (NumberOfChildren == 0) {
1225     Status = gBS->OpenProtocol (
1226                     Controller,
1227                     &gEfiNvmExpressPassThruProtocolGuid,
1228                     (VOID **)&PassThru,
1229                     This->DriverBindingHandle,
1230                     Controller,
1231                     EFI_OPEN_PROTOCOL_GET_PROTOCOL
1232                     );
1233
1234     if (!EFI_ERROR (Status)) {
1235       Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (PassThru);
1236
1237       //
1238       // Wait for the asynchronous PassThru queue to become empty.
1239       //
1240       while (TRUE) {
1241         OldTpl  = gBS->RaiseTPL (TPL_NOTIFY);
1242         IsEmpty = IsListEmpty (&Private->AsyncPassThruQueue) &&
1243                   IsListEmpty (&Private->UnsubmittedSubtasks);
1244         gBS->RestoreTPL (OldTpl);
1245
1246         if (IsEmpty) {
1247           break;
1248         }
1249
1250         gBS->Stall (100);
1251       }
1252
1253       gBS->UninstallMultipleProtocolInterfaces (
1254              Controller,
1255              &gEfiNvmExpressPassThruProtocolGuid,
1256              PassThru,
1257              NULL
1258              );
1259
1260       if (Private->TimerEvent != NULL) {
1261         gBS->CloseEvent (Private->TimerEvent);
1262       }
1263
1264       if (Private->Mapping != NULL) {
1265         Private->PciIo->Unmap (Private->PciIo, Private->Mapping);
1266       }
1267
1268       if (Private->Buffer != NULL) {
1269         Private->PciIo->FreeBuffer (Private->PciIo, 6, Private->Buffer);
1270       }
1271
1272       FreePool (Private->ControllerData);
1273       FreePool (Private);
1274     }
1275
1276     gBS->CloseProtocol (
1277            Controller,
1278            &gEfiPciIoProtocolGuid,
1279            This->DriverBindingHandle,
1280            Controller
1281            );
1282     gBS->CloseProtocol (
1283            Controller,
1284            &gEfiDevicePathProtocolGuid,
1285            This->DriverBindingHandle,
1286            Controller
1287            );
1288
1289     NvmeUnregisterShutdownNotification ();
1290
1291     return EFI_SUCCESS;
1292   }
1293
1294   AllChildrenStopped = TRUE;
1295
1296   for (Index = 0; Index < NumberOfChildren; Index++) {
1297     Status = UnregisterNvmeNamespace (This, Controller, ChildHandleBuffer[Index]);
1298     if (EFI_ERROR (Status)) {
1299       AllChildrenStopped = FALSE;
1300     }
1301   }
1302
1303   if (!AllChildrenStopped) {
1304     return EFI_DEVICE_ERROR;
1305   }
1306
1307   return EFI_SUCCESS;
1308 }
1309
1310 /**
1311   This is the unload handle for the NVM Express driver.
1312
1313   Disconnect the driver specified by ImageHandle from the NVMe device in the handle database.
1314   Uninstall all the protocols installed in the driver.
1315
1316   @param[in]  ImageHandle       The drivers' driver image.
1317
1318   @retval EFI_SUCCESS           The image is unloaded.
1319   @retval Others                Failed to unload the image.
1320
1321 **/
1322 EFI_STATUS
1323 EFIAPI
1324 NvmExpressUnload (
1325   IN EFI_HANDLE  ImageHandle
1326   )
1327 {
1328   EFI_STATUS                    Status;
1329   EFI_HANDLE                    *DeviceHandleBuffer;
1330   UINTN                         DeviceHandleCount;
1331   UINTN                         Index;
1332   EFI_COMPONENT_NAME_PROTOCOL   *ComponentName;
1333   EFI_COMPONENT_NAME2_PROTOCOL  *ComponentName2;
1334
1335   //
1336   // Get the list of the device handles managed by this driver.
1337   // If there is an error getting the list, then means the driver
1338   // doesn't manage any device. At this way, we would only close
1339   // those protocols installed at image handle.
1340   //
1341   DeviceHandleBuffer = NULL;
1342   Status             = gBS->LocateHandleBuffer (
1343                               ByProtocol,
1344                               &gEfiNvmExpressPassThruProtocolGuid,
1345                               NULL,
1346                               &DeviceHandleCount,
1347                               &DeviceHandleBuffer
1348                               );
1349
1350   if (!EFI_ERROR (Status)) {
1351     //
1352     // Disconnect the driver specified by ImageHandle from all
1353     // the devices in the handle database.
1354     //
1355     for (Index = 0; Index < DeviceHandleCount; Index++) {
1356       Status = gBS->DisconnectController (
1357                       DeviceHandleBuffer[Index],
1358                       ImageHandle,
1359                       NULL
1360                       );
1361       if (EFI_ERROR (Status)) {
1362         goto EXIT;
1363       }
1364     }
1365   }
1366
1367   //
1368   // Uninstall all the protocols installed in the driver entry point
1369   //
1370   Status = gBS->UninstallMultipleProtocolInterfaces (
1371                   ImageHandle,
1372                   &gEfiDriverBindingProtocolGuid,
1373                   &gNvmExpressDriverBinding,
1374                   &gEfiDriverSupportedEfiVersionProtocolGuid,
1375                   &gNvmExpressDriverSupportedEfiVersion,
1376                   NULL
1377                   );
1378
1379   if (EFI_ERROR (Status)) {
1380     goto EXIT;
1381   }
1382
1383   //
1384   // Note we have to one by one uninstall the following protocols.
1385   // It's because some of them are optionally installed based on
1386   // the following PCD settings.
1387   //   gEfiMdePkgTokenSpaceGuid.PcdDriverDiagnosticsDisable
1388   //   gEfiMdePkgTokenSpaceGuid.PcdComponentNameDisable
1389   //   gEfiMdePkgTokenSpaceGuid.PcdDriverDiagnostics2Disable
1390   //   gEfiMdePkgTokenSpaceGuid.PcdComponentName2Disable
1391   //
1392   Status = gBS->HandleProtocol (
1393                   ImageHandle,
1394                   &gEfiComponentNameProtocolGuid,
1395                   (VOID **)&ComponentName
1396                   );
1397   if (!EFI_ERROR (Status)) {
1398     gBS->UninstallProtocolInterface (
1399            ImageHandle,
1400            &gEfiComponentNameProtocolGuid,
1401            ComponentName
1402            );
1403   }
1404
1405   Status = gBS->HandleProtocol (
1406                   ImageHandle,
1407                   &gEfiComponentName2ProtocolGuid,
1408                   (VOID **)&ComponentName2
1409                   );
1410   if (!EFI_ERROR (Status)) {
1411     gBS->UninstallProtocolInterface (
1412            ImageHandle,
1413            &gEfiComponentName2ProtocolGuid,
1414            ComponentName2
1415            );
1416   }
1417
1418   Status = EFI_SUCCESS;
1419
1420 EXIT:
1421   //
1422   // Free the buffer containing the list of handles from the handle database
1423   //
1424   if (DeviceHandleBuffer != NULL) {
1425     gBS->FreePool (DeviceHandleBuffer);
1426   }
1427
1428   return Status;
1429 }
1430
1431 /**
1432   The entry point for Nvm Express driver, used to install Nvm Express driver on the ImageHandle.
1433
1434   @param  ImageHandle   The firmware allocated handle for this driver image.
1435   @param  SystemTable   Pointer to the EFI system table.
1436
1437   @retval EFI_SUCCESS   Driver loaded.
1438   @retval other         Driver not loaded.
1439
1440 **/
1441 EFI_STATUS
1442 EFIAPI
1443 NvmExpressDriverEntry (
1444   IN EFI_HANDLE        ImageHandle,
1445   IN EFI_SYSTEM_TABLE  *SystemTable
1446   )
1447 {
1448   EFI_STATUS  Status;
1449
1450   Status = EfiLibInstallDriverBindingComponentName2 (
1451              ImageHandle,
1452              SystemTable,
1453              &gNvmExpressDriverBinding,
1454              ImageHandle,
1455              &gNvmExpressComponentName,
1456              &gNvmExpressComponentName2
1457              );
1458   ASSERT_EFI_ERROR (Status);
1459
1460   //
1461   // Install EFI Driver Supported EFI Version Protocol required for
1462   // EFI drivers that are on PCI and other plug in cards.
1463   //
1464   gNvmExpressDriverSupportedEfiVersion.FirmwareVersion = 0x00020028;
1465   Status                                               = gBS->InstallMultipleProtocolInterfaces (
1466                                                                 &ImageHandle,
1467                                                                 &gEfiDriverSupportedEfiVersionProtocolGuid,
1468                                                                 &gNvmExpressDriverSupportedEfiVersion,
1469                                                                 NULL
1470                                                                 );
1471   ASSERT_EFI_ERROR (Status);
1472   return Status;
1473 }