MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.c

   1 /** @file
   2   NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows
   3   NVM Express specification.
   4
   5   Copyright (c) 2013 - 2017, Intel Corporation. All rights reserved.<BR>
   6   This program and the accompanying materials
   7   are licensed and made available under the terms and conditions of the BSD License
   8   which accompanies this distribution.  The full text of the license may be found at
   9   http://opensource.org/licenses/bsd-license.php.
  10
  11   THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  12   WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  13
  14 **/
  15
  16 #include "NvmExpress.h"
  17
  18 //
  19 // NVM Express Driver Binding Protocol Instance
  20 //
  21 EFI_DRIVER_BINDING_PROTOCOL gNvmExpressDriverBinding = {
  22   NvmExpressDriverBindingSupported,
  23   NvmExpressDriverBindingStart,
  24   NvmExpressDriverBindingStop,
  25   0x10,
  26   NULL,
  27   NULL
  28 };
  29
  30 //
  31 // NVM Express EFI Driver Supported EFI Version Protocol Instance
  32 //
  33 EFI_DRIVER_SUPPORTED_EFI_VERSION_PROTOCOL gNvmExpressDriverSupportedEfiVersion = {
  34   sizeof (EFI_DRIVER_SUPPORTED_EFI_VERSION_PROTOCOL), // Size of Protocol structure.
  35   0                                                   // Version number to be filled at start up.
  36 };
  37
  38 //
  39 // Template for NVM Express Pass Thru Mode data structure.
  40 //
  41 GLOBAL_REMOVE_IF_UNREFERENCED EFI_NVM_EXPRESS_PASS_THRU_MODE gEfiNvmExpressPassThruMode = {
  42   EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL   |
  43   EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL    |
  44   EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_NONBLOCKIO |
  45   EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_CMD_SET_NVM,
  46   sizeof (UINTN),
  47   0x10100
  48 };
  49
  50 /**
  51   Check if the specified Nvm Express device namespace is active, and create child handles
  52   for them with BlockIo and DiskInfo protocol instances.
  53
  54   @param[in] Private         The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
  55   @param[in] NamespaceId     The NVM Express namespace ID  for which a device path node is to be
  56                              allocated and built. Caller must set the NamespaceId to zero if the
  57                              device path node will contain a valid UUID.
  58
  59   @retval EFI_SUCCESS        All the namespaces in the device are successfully enumerated.
  60   @return Others             Some error occurs when enumerating the namespaces.
  61
  62 **/
  63 EFI_STATUS
  64 EnumerateNvmeDevNamespace (
  65   IN NVME_CONTROLLER_PRIVATE_DATA       *Private,
  66   UINT32                                NamespaceId
  67   )
  68 {
  69   NVME_ADMIN_NAMESPACE_DATA             *NamespaceData;
  70   EFI_DEVICE_PATH_PROTOCOL              *NewDevicePathNode;
  71   EFI_DEVICE_PATH_PROTOCOL              *DevicePath;
  72   EFI_HANDLE                            DeviceHandle;
  73   EFI_DEVICE_PATH_PROTOCOL              *ParentDevicePath;
  74   EFI_DEVICE_PATH_PROTOCOL              *RemainingDevicePath;
  75   NVME_DEVICE_PRIVATE_DATA              *Device;
  76   EFI_STATUS                            Status;
  77   UINT32                                Lbads;
  78   UINT32                                Flbas;
  79   UINT32                                LbaFmtIdx;
  80   UINT8                                 Sn[21];
  81   UINT8                                 Mn[41];
  82   VOID                                  *DummyInterface;
  83
  84   NewDevicePathNode = NULL;
  85   DevicePath        = NULL;
  86   Device            = NULL;
  87
  88   //
  89   // Allocate a buffer for Identify Namespace data
  90   //
  91   NamespaceData = AllocateZeroPool(sizeof (NVME_ADMIN_NAMESPACE_DATA));
  92   if(NamespaceData == NULL) {
  93     return EFI_OUT_OF_RESOURCES;
  94   }
  95
  96   ParentDevicePath = Private->ParentDevicePath;
  97   //
  98   // Identify Namespace
  99   //
 100   Status = NvmeIdentifyNamespace (
 101              Private,
 102              NamespaceId,
 103              (VOID *)NamespaceData
 104              );
 105   if (EFI_ERROR(Status)) {
 106     goto Exit;
 107   }
 108   //
 109   // Validate Namespace
 110   //
 111   if (NamespaceData->Ncap == 0) {
 112     Status = EFI_DEVICE_ERROR;
 113   } else {
 114     //
 115     // allocate device private data for each discovered namespace
 116     //
 117     Device = AllocateZeroPool(sizeof(NVME_DEVICE_PRIVATE_DATA));
 118     if (Device == NULL) {
 119       Status = EFI_OUT_OF_RESOURCES;
 120       goto Exit;
 121     }
 122
 123     //
 124     // Initialize SSD namespace instance data
 125     //
 126     Device->Signature           = NVME_DEVICE_PRIVATE_DATA_SIGNATURE;
 127     Device->NamespaceId         = NamespaceId;
 128     Device->NamespaceUuid       = NamespaceData->Eui64;
 129
 130     Device->ControllerHandle    = Private->ControllerHandle;
 131     Device->DriverBindingHandle = Private->DriverBindingHandle;
 132     Device->Controller          = Private;
 133
 134     //
 135     // Build BlockIo media structure
 136     //
 137     Device->Media.MediaId        = 0;
 138     Device->Media.RemovableMedia = FALSE;
 139     Device->Media.MediaPresent   = TRUE;
 140     Device->Media.LogicalPartition = FALSE;
 141     Device->Media.ReadOnly       = FALSE;
 142     Device->Media.WriteCaching   = FALSE;
 143     Device->Media.IoAlign        = Private->PassThruMode.IoAlign;
 144
 145     Flbas     = NamespaceData->Flbas;
 146     LbaFmtIdx = Flbas & 0xF;
 147     Lbads     = NamespaceData->LbaFormat[LbaFmtIdx].Lbads;
 148     Device->Media.BlockSize = (UINT32)1 << Lbads;
 149
 150     Device->Media.LastBlock                     = NamespaceData->Nsze - 1;
 151     Device->Media.LogicalBlocksPerPhysicalBlock = 1;
 152     Device->Media.LowestAlignedLba              = 1;
 153
 154     //
 155     // Create BlockIo Protocol instance
 156     //
 157     Device->BlockIo.Revision     = EFI_BLOCK_IO_PROTOCOL_REVISION2;
 158     Device->BlockIo.Media        = &Device->Media;
 159     Device->BlockIo.Reset        = NvmeBlockIoReset;
 160     Device->BlockIo.ReadBlocks   = NvmeBlockIoReadBlocks;
 161     Device->BlockIo.WriteBlocks  = NvmeBlockIoWriteBlocks;
 162     Device->BlockIo.FlushBlocks  = NvmeBlockIoFlushBlocks;
 163
 164     //
 165     // Create BlockIo2 Protocol instance
 166     //
 167     Device->BlockIo2.Media          = &Device->Media;
 168     Device->BlockIo2.Reset          = NvmeBlockIoResetEx;
 169     Device->BlockIo2.ReadBlocksEx   = NvmeBlockIoReadBlocksEx;
 170     Device->BlockIo2.WriteBlocksEx  = NvmeBlockIoWriteBlocksEx;
 171     Device->BlockIo2.FlushBlocksEx  = NvmeBlockIoFlushBlocksEx;
 172     InitializeListHead (&Device->AsyncQueue);
 173
 174     //
 175     // Create StorageSecurityProtocol Instance
 176     //
 177     Device->StorageSecurity.ReceiveData = NvmeStorageSecurityReceiveData;
 178     Device->StorageSecurity.SendData    = NvmeStorageSecuritySendData;
 179
 180     //
 181     // Create DiskInfo Protocol instance
 182     //
 183     CopyMem (&Device->NamespaceData, NamespaceData, sizeof (NVME_ADMIN_NAMESPACE_DATA));
 184     InitializeDiskInfo (Device);
 185
 186     //
 187     // Create a Nvm Express Namespace Device Path Node
 188     //
 189     Status = Private->Passthru.BuildDevicePath (
 190                                  &Private->Passthru,
 191                                  Device->NamespaceId,
 192                                  &NewDevicePathNode
 193                                  );
 194
 195     if (EFI_ERROR(Status)) {
 196       goto Exit;
 197     }
 198
 199     //
 200     // Append the SSD node to the controller's device path
 201     //
 202     DevicePath = AppendDevicePathNode (ParentDevicePath, NewDevicePathNode);
 203     if (DevicePath == NULL) {
 204       Status = EFI_OUT_OF_RESOURCES;
 205       goto Exit;
 206     }
 207
 208     DeviceHandle = NULL;
 209     RemainingDevicePath = DevicePath;
 210     Status = gBS->LocateDevicePath (&gEfiDevicePathProtocolGuid, &RemainingDevicePath, &DeviceHandle);
 211     if (!EFI_ERROR (Status) && (DeviceHandle != NULL) && IsDevicePathEnd(RemainingDevicePath)) {
 212       Status = EFI_ALREADY_STARTED;
 213       FreePool (DevicePath);
 214       goto Exit;
 215     }
 216
 217     Device->DevicePath = DevicePath;
 218
 219     //
 220     // Make sure the handle is NULL so we create a new handle
 221     //
 222     Device->DeviceHandle = NULL;
 223
 224     Status = gBS->InstallMultipleProtocolInterfaces (
 225                     &Device->DeviceHandle,
 226                     &gEfiDevicePathProtocolGuid,
 227                     Device->DevicePath,
 228                     &gEfiBlockIoProtocolGuid,
 229                     &Device->BlockIo,
 230                     &gEfiBlockIo2ProtocolGuid,
 231                     &Device->BlockIo2,
 232                     &gEfiDiskInfoProtocolGuid,
 233                     &Device->DiskInfo,
 234                     NULL
 235                     );
 236
 237     if(EFI_ERROR(Status)) {
 238       goto Exit;
 239     }
 240
 241     //
 242     // Check if the NVMe controller supports the Security Send and Security Receive commands
 243     //
 244     if ((Private->ControllerData->Oacs & SECURITY_SEND_RECEIVE_SUPPORTED) != 0) {
 245       Status = gBS->InstallProtocolInterface (
 246                       &Device->DeviceHandle,
 247                       &gEfiStorageSecurityCommandProtocolGuid,
 248                       EFI_NATIVE_INTERFACE,
 249                       &Device->StorageSecurity
 250                       );
 251       if(EFI_ERROR(Status)) {
 252         gBS->UninstallMultipleProtocolInterfaces (
 253                &Device->DeviceHandle,
 254                &gEfiDevicePathProtocolGuid,
 255                Device->DevicePath,
 256                &gEfiBlockIoProtocolGuid,
 257                &Device->BlockIo,
 258                &gEfiBlockIo2ProtocolGuid,
 259                &Device->BlockIo2,
 260                &gEfiDiskInfoProtocolGuid,
 261                &Device->DiskInfo,
 262                NULL
 263                );
 264         goto Exit;
 265       }
 266     }
 267
 268     gBS->OpenProtocol (
 269            Private->ControllerHandle,
 270            &gEfiNvmExpressPassThruProtocolGuid,
 271            (VOID **) &DummyInterface,
 272            Private->DriverBindingHandle,
 273            Device->DeviceHandle,
 274            EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER
 275            );
 276
 277     //
 278     // Dump NvmExpress Identify Namespace Data
 279     //
 280     DEBUG ((EFI_D_INFO, " == NVME IDENTIFY NAMESPACE [%d] DATA ==\n", NamespaceId));
 281     DEBUG ((EFI_D_INFO, "    NSZE        : 0x%x\n", NamespaceData->Nsze));
 282     DEBUG ((EFI_D_INFO, "    NCAP        : 0x%x\n", NamespaceData->Ncap));
 283     DEBUG ((EFI_D_INFO, "    NUSE        : 0x%x\n", NamespaceData->Nuse));
 284     DEBUG ((EFI_D_INFO, "    LBAF0.LBADS : 0x%x\n", (NamespaceData->LbaFormat[0].Lbads)));
 285
 286     //
 287     // Build controller name for Component Name (2) protocol.
 288     //
 289     CopyMem (Sn, Private->ControllerData->Sn, sizeof (Private->ControllerData->Sn));
 290     Sn[20] = 0;
 291     CopyMem (Mn, Private->ControllerData->Mn, sizeof (Private->ControllerData->Mn));
 292     Mn[40] = 0;
 293     UnicodeSPrintAsciiFormat (Device->ModelName, sizeof (Device->ModelName), "%a-%a-%x", Sn, Mn, NamespaceData->Eui64);
 294
 295     AddUnicodeString2 (
 296       "eng",
 297       gNvmExpressComponentName.SupportedLanguages,
 298       &Device->ControllerNameTable,
 299       Device->ModelName,
 300       TRUE
 301       );
 302
 303     AddUnicodeString2 (
 304       "en",
 305       gNvmExpressComponentName2.SupportedLanguages,
 306       &Device->ControllerNameTable,
 307       Device->ModelName,
 308       FALSE
 309       );
 310   }
 311
 312 Exit:
 313   if(NamespaceData != NULL) {
 314     FreePool (NamespaceData);
 315   }
 316
 317   if (NewDevicePathNode != NULL) {
 318     FreePool (NewDevicePathNode);
 319   }
 320
 321   if(EFI_ERROR(Status) && (Device != NULL) && (Device->DevicePath != NULL)) {
 322     FreePool (Device->DevicePath);
 323   }
 324   if(EFI_ERROR(Status) && (Device != NULL)) {
 325     FreePool (Device);
 326   }
 327   return Status;
 328 }
 329
 330 /**
 331   Discover all Nvm Express device namespaces, and create child handles for them with BlockIo
 332   and DiskInfo protocol instances.
 333
 334   @param[in] Private         The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 335
 336   @retval EFI_SUCCESS        All the namespaces in the device are successfully enumerated.
 337   @return Others             Some error occurs when enumerating the namespaces.
 338
 339 **/
 340 EFI_STATUS
 341 DiscoverAllNamespaces (
 342   IN NVME_CONTROLLER_PRIVATE_DATA       *Private
 343   )
 344 {
 345   EFI_STATUS                            Status;
 346   UINT32                                NamespaceId;
 347   EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL    *Passthru;
 348
 349   NamespaceId   = 0xFFFFFFFF;
 350   Passthru      = &Private->Passthru;
 351
 352   while (TRUE) {
 353     Status = Passthru->GetNextNamespace (
 354                          Passthru,
 355                          (UINT32 *)&NamespaceId
 356                          );
 357
 358     if (EFI_ERROR (Status)) {
 359       break;
 360     }
 361
 362     Status = EnumerateNvmeDevNamespace (
 363                Private,
 364                NamespaceId
 365                );
 366
 367     if (EFI_ERROR(Status)) {
 368       continue;
 369     }
 370   }
 371
 372   return EFI_SUCCESS;
 373 }
 374
 375 /**
 376   Unregisters a Nvm Express device namespace.
 377
 378   This function removes the protocols installed on the controller handle and
 379   frees the resources allocated for the namespace.
 380
 381   @param  This                  The pointer to EFI_DRIVER_BINDING_PROTOCOL instance.
 382   @param  Controller            The controller handle of the namespace.
 383   @param  Handle                The child handle.
 384
 385   @retval EFI_SUCCESS           The namespace is successfully unregistered.
 386   @return Others                Some error occurs when unregistering the namespace.
 387
 388 **/
 389 EFI_STATUS
 390 UnregisterNvmeNamespace (
 391   IN  EFI_DRIVER_BINDING_PROTOCOL    *This,
 392   IN  EFI_HANDLE                     Controller,
 393   IN  EFI_HANDLE                     Handle
 394   )
 395 {
 396   EFI_STATUS                               Status;
 397   EFI_BLOCK_IO_PROTOCOL                    *BlockIo;
 398   NVME_DEVICE_PRIVATE_DATA                 *Device;
 399   EFI_STORAGE_SECURITY_COMMAND_PROTOCOL    *StorageSecurity;
 400   BOOLEAN                                  IsEmpty;
 401   EFI_TPL                                  OldTpl;
 402   VOID                                     *DummyInterface;
 403
 404   BlockIo = NULL;
 405
 406   Status = gBS->OpenProtocol (
 407                   Handle,
 408                   &gEfiBlockIoProtocolGuid,
 409                   (VOID **) &BlockIo,
 410                   This->DriverBindingHandle,
 411                   Controller,
 412                   EFI_OPEN_PROTOCOL_GET_PROTOCOL
 413                   );
 414   if (EFI_ERROR (Status)) {
 415     return Status;
 416   }
 417
 418   Device  = NVME_DEVICE_PRIVATE_DATA_FROM_BLOCK_IO (BlockIo);
 419
 420   //
 421   // Wait for the device's asynchronous I/O queue to become empty.
 422   //
 423   while (TRUE) {
 424     OldTpl  = gBS->RaiseTPL (TPL_NOTIFY);
 425     IsEmpty = IsListEmpty (&Device->AsyncQueue);
 426     gBS->RestoreTPL (OldTpl);
 427
 428     if (IsEmpty) {
 429       break;
 430     }
 431
 432     gBS->Stall (100);
 433   }
 434
 435   //
 436   // Close the child handle
 437   //
 438   gBS->CloseProtocol (
 439          Controller,
 440          &gEfiNvmExpressPassThruProtocolGuid,
 441          This->DriverBindingHandle,
 442          Handle
 443          );
 444
 445   //
 446   // The Nvm Express driver installs the BlockIo and DiskInfo in the DriverBindingStart().
 447   // Here should uninstall both of them.
 448   //
 449   Status = gBS->UninstallMultipleProtocolInterfaces (
 450                   Handle,
 451                   &gEfiDevicePathProtocolGuid,
 452                   Device->DevicePath,
 453                   &gEfiBlockIoProtocolGuid,
 454                   &Device->BlockIo,
 455                   &gEfiBlockIo2ProtocolGuid,
 456                   &Device->BlockIo2,
 457                   &gEfiDiskInfoProtocolGuid,
 458                   &Device->DiskInfo,
 459                   NULL
 460                   );
 461
 462   if (EFI_ERROR (Status)) {
 463     gBS->OpenProtocol (
 464            Controller,
 465            &gEfiNvmExpressPassThruProtocolGuid,
 466            (VOID **) &DummyInterface,
 467            This->DriverBindingHandle,
 468            Handle,
 469            EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER
 470            );
 471     return Status;
 472   }
 473
 474   //
 475   // If Storage Security Command Protocol is installed, then uninstall this protocol.
 476   //
 477   Status = gBS->OpenProtocol (
 478                   Handle,
 479                   &gEfiStorageSecurityCommandProtocolGuid,
 480                   (VOID **) &StorageSecurity,
 481                   This->DriverBindingHandle,
 482                   Controller,
 483                   EFI_OPEN_PROTOCOL_GET_PROTOCOL
 484                   );
 485
 486   if (!EFI_ERROR (Status)) {
 487     Status = gBS->UninstallProtocolInterface (
 488                     Handle,
 489                     &gEfiStorageSecurityCommandProtocolGuid,
 490                     &Device->StorageSecurity
 491                     );
 492     if (EFI_ERROR (Status)) {
 493       gBS->OpenProtocol (
 494         Controller,
 495         &gEfiNvmExpressPassThruProtocolGuid,
 496         (VOID **) &DummyInterface,
 497         This->DriverBindingHandle,
 498         Handle,
 499         EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER
 500         );
 501       return Status;
 502     }
 503   }
 504
 505   if(Device->DevicePath != NULL) {
 506     FreePool (Device->DevicePath);
 507   }
 508
 509   if (Device->ControllerNameTable != NULL) {
 510     FreeUnicodeStringTable (Device->ControllerNameTable);
 511   }
 512
 513   FreePool (Device);
 514
 515   return EFI_SUCCESS;
 516 }
 517
 518 /**
 519   Call back function when the timer event is signaled.
 520
 521   @param[in]  Event     The Event this notify function registered to.
 522   @param[in]  Context   Pointer to the context data registered to the
 523                         Event.
 524
 525 **/
 526 VOID
 527 EFIAPI
 528 ProcessAsyncTaskList (
 529   IN EFI_EVENT                    Event,
 530   IN VOID*                        Context
 531   )
 532 {
 533   NVME_CONTROLLER_PRIVATE_DATA         *Private;
 534   EFI_PCI_IO_PROTOCOL                  *PciIo;
 535   NVME_CQ                              *Cq;
 536   UINT16                               QueueId;
 537   UINT32                               Data;
 538   LIST_ENTRY                           *Link;
 539   LIST_ENTRY                           *NextLink;
 540   NVME_PASS_THRU_ASYNC_REQ             *AsyncRequest;
 541   NVME_BLKIO2_SUBTASK                  *Subtask;
 542   NVME_BLKIO2_REQUEST                  *BlkIo2Request;
 543   EFI_BLOCK_IO2_TOKEN                  *Token;
 544   BOOLEAN                              HasNewItem;
 545   EFI_STATUS                           Status;
 546
 547   Private    = (NVME_CONTROLLER_PRIVATE_DATA*)Context;
 548   QueueId    = 2;
 549   Cq         = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
 550   HasNewItem = FALSE;
 551   PciIo      = Private->PciIo;
 552
 553   //
 554   // Submit asynchronous subtasks to the NVMe Submission Queue
 555   //
 556   for (Link = GetFirstNode (&Private->UnsubmittedSubtasks);
 557        !IsNull (&Private->UnsubmittedSubtasks, Link);
 558        Link = NextLink) {
 559     NextLink      = GetNextNode (&Private->UnsubmittedSubtasks, Link);
 560     Subtask       = NVME_BLKIO2_SUBTASK_FROM_LINK (Link);
 561     BlkIo2Request = Subtask->BlockIo2Request;
 562     Token         = BlkIo2Request->Token;
 563     RemoveEntryList (Link);
 564     BlkIo2Request->UnsubmittedSubtaskNum--;
 565
 566     //
 567     // If any previous subtask fails, do not process subsequent ones.
 568     //
 569     if (Token->TransactionStatus != EFI_SUCCESS) {
 570       if (IsListEmpty (&BlkIo2Request->SubtasksQueue) &&
 571           BlkIo2Request->LastSubtaskSubmitted &&
 572           (BlkIo2Request->UnsubmittedSubtaskNum == 0)) {
 573         //
 574         // Remove the BlockIo2 request from the device asynchronous queue.
 575         //
 576         RemoveEntryList (&BlkIo2Request->Link);
 577         FreePool (BlkIo2Request);
 578         gBS->SignalEvent (Token->Event);
 579       }
 580
 581       FreePool (Subtask->CommandPacket->NvmeCmd);
 582       FreePool (Subtask->CommandPacket->NvmeCompletion);
 583       FreePool (Subtask->CommandPacket);
 584       FreePool (Subtask);
 585
 586       continue;
 587     }
 588
 589     Status = Private->Passthru.PassThru (
 590                                  &Private->Passthru,
 591                                  Subtask->NamespaceId,
 592                                  Subtask->CommandPacket,
 593                                  Subtask->Event
 594                                  );
 595     if (Status == EFI_NOT_READY) {
 596       InsertHeadList (&Private->UnsubmittedSubtasks, Link);
 597       BlkIo2Request->UnsubmittedSubtaskNum++;
 598       break;
 599     } else if (EFI_ERROR (Status)) {
 600       Token->TransactionStatus = EFI_DEVICE_ERROR;
 601
 602       if (IsListEmpty (&BlkIo2Request->SubtasksQueue) &&
 603           Subtask->IsLast) {
 604         //
 605         // Remove the BlockIo2 request from the device asynchronous queue.
 606         //
 607         RemoveEntryList (&BlkIo2Request->Link);
 608         FreePool (BlkIo2Request);
 609         gBS->SignalEvent (Token->Event);
 610       }
 611
 612       FreePool (Subtask->CommandPacket->NvmeCmd);
 613       FreePool (Subtask->CommandPacket->NvmeCompletion);
 614       FreePool (Subtask->CommandPacket);
 615       FreePool (Subtask);
 616     } else {
 617       InsertTailList (&BlkIo2Request->SubtasksQueue, Link);
 618       if (Subtask->IsLast) {
 619         BlkIo2Request->LastSubtaskSubmitted = TRUE;
 620       }
 621     }
 622   }
 623
 624   while (Cq->Pt != Private->Pt[QueueId]) {
 625     ASSERT (Cq->Sqid == QueueId);
 626
 627     HasNewItem = TRUE;
 628
 629     //
 630     // Find the command with given Command Id.
 631     //
 632     for (Link = GetFirstNode (&Private->AsyncPassThruQueue);
 633          !IsNull (&Private->AsyncPassThruQueue, Link);
 634          Link = NextLink) {
 635       NextLink = GetNextNode (&Private->AsyncPassThruQueue, Link);
 636       AsyncRequest = NVME_PASS_THRU_ASYNC_REQ_FROM_THIS (Link);
 637       if (AsyncRequest->CommandId == Cq->Cid) {
 638         //
 639         // Copy the Respose Queue entry for this command to the callers
 640         // response buffer.
 641         //
 642         CopyMem (
 643           AsyncRequest->Packet->NvmeCompletion,
 644           Cq,
 645           sizeof(EFI_NVM_EXPRESS_COMPLETION)
 646           );
 647
 648         //
 649         // Free the resources allocated before cmd submission
 650         //
 651         if (AsyncRequest->MapData != NULL) {
 652           PciIo->Unmap (PciIo, AsyncRequest->MapData);
 653         }
 654         if (AsyncRequest->MapMeta != NULL) {
 655           PciIo->Unmap (PciIo, AsyncRequest->MapMeta);
 656         }
 657         if (AsyncRequest->MapPrpList != NULL) {
 658           PciIo->Unmap (PciIo, AsyncRequest->MapPrpList);
 659         }
 660         if (AsyncRequest->PrpListHost != NULL) {
 661           PciIo->FreeBuffer (
 662                    PciIo,
 663                    AsyncRequest->PrpListNo,
 664                    AsyncRequest->PrpListHost
 665                    );
 666         }
 667
 668         RemoveEntryList (Link);
 669         gBS->SignalEvent (AsyncRequest->CallerEvent);
 670         FreePool (AsyncRequest);
 671
 672         //
 673         // Update submission queue head.
 674         //
 675         Private->AsyncSqHead = Cq->Sqhd;
 676         break;
 677       }
 678     }
 679
 680     Private->CqHdbl[QueueId].Cqh++;
 681     if (Private->CqHdbl[QueueId].Cqh > NVME_ASYNC_CCQ_SIZE) {
 682       Private->CqHdbl[QueueId].Cqh = 0;
 683       Private->Pt[QueueId] ^= 1;
 684     }
 685
 686     Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
 687   }
 688
 689   if (HasNewItem) {
 690     Data  = ReadUnaligned32 ((UINT32*)&Private->CqHdbl[QueueId]);
 691     PciIo->Mem.Write (
 692                  PciIo,
 693                  EfiPciIoWidthUint32,
 694                  NVME_BAR,
 695                  NVME_CQHDBL_OFFSET(QueueId, Private->Cap.Dstrd),
 696                  1,
 697                  &Data
 698                  );
 699   }
 700 }
 701
 702 /**
 703   Tests to see if this driver supports a given controller. If a child device is provided,
 704   it further tests to see if this driver supports creating a handle for the specified child device.
 705
 706   This function checks to see if the driver specified by This supports the device specified by
 707   ControllerHandle. Drivers will typically use the device path attached to
 708   ControllerHandle and/or the services from the bus I/O abstraction attached to
 709   ControllerHandle to determine if the driver supports ControllerHandle. This function
 710   may be called many times during platform initialization. In order to reduce boot times, the tests
 711   performed by this function must be very small, and take as little time as possible to execute. This
 712   function must not change the state of any hardware devices, and this function must be aware that the
 713   device specified by ControllerHandle may already be managed by the same driver or a
 714   different driver. This function must match its calls to AllocatePages() with FreePages(),
 715   AllocatePool() with FreePool(), and OpenProtocol() with CloseProtocol().
 716   Since ControllerHandle may have been previously started by the same driver, if a protocol is
 717   already in the opened state, then it must not be closed with CloseProtocol(). This is required
 718   to guarantee the state of ControllerHandle is not modified by this function.
 719
 720   @param[in]  This                 A pointer to the EFI_DRIVER_BINDING_PROTOCOL instance.
 721   @param[in]  ControllerHandle     The handle of the controller to test. This handle
 722                                    must support a protocol interface that supplies
 723                                    an I/O abstraction to the driver.
 724   @param[in]  RemainingDevicePath  A pointer to the remaining portion of a device path.  This
 725                                    parameter is ignored by device drivers, and is optional for bus
 726                                    drivers. For bus drivers, if this parameter is not NULL, then
 727                                    the bus driver must determine if the bus controller specified
 728                                    by ControllerHandle and the child controller specified
 729                                    by RemainingDevicePath are both supported by this
 730                                    bus driver.
 731
 732   @retval EFI_SUCCESS              The device specified by ControllerHandle and
 733                                    RemainingDevicePath is supported by the driver specified by This.
 734   @retval EFI_ALREADY_STARTED      The device specified by ControllerHandle and
 735                                    RemainingDevicePath is already being managed by the driver
 736                                    specified by This.
 737   @retval EFI_ACCESS_DENIED        The device specified by ControllerHandle and
 738                                    RemainingDevicePath is already being managed by a different
 739                                    driver or an application that requires exclusive access.
 740                                    Currently not implemented.
 741   @retval EFI_UNSUPPORTED          The device specified by ControllerHandle and
 742                                    RemainingDevicePath is not supported by the driver specified by This.
 743 **/
 744 EFI_STATUS
 745 EFIAPI
 746 NvmExpressDriverBindingSupported (
 747   IN EFI_DRIVER_BINDING_PROTOCOL  *This,
 748   IN EFI_HANDLE                   Controller,
 749   IN EFI_DEVICE_PATH_PROTOCOL     *RemainingDevicePath
 750   )
 751 {
 752   EFI_STATUS                Status;
 753   EFI_DEV_PATH_PTR          DevicePathNode;
 754   EFI_DEVICE_PATH_PROTOCOL  *ParentDevicePath;
 755   EFI_PCI_IO_PROTOCOL       *PciIo;
 756   UINT8                     ClassCode[3];
 757
 758   //
 759   // Check whether device path is valid
 760   //
 761   if (RemainingDevicePath != NULL) {
 762     //
 763     // Check if RemainingDevicePath is the End of Device Path Node,
 764     // if yes, go on checking other conditions
 765     //
 766     if (!IsDevicePathEnd (RemainingDevicePath)) {
 767       //
 768       // If RemainingDevicePath isn't the End of Device Path Node,
 769       // check its validation
 770       //
 771       DevicePathNode.DevPath = RemainingDevicePath;
 772
 773       if ((DevicePathNode.DevPath->Type    != MESSAGING_DEVICE_PATH) ||
 774           (DevicePathNode.DevPath->SubType != MSG_NVME_NAMESPACE_DP) ||
 775           (DevicePathNodeLength(DevicePathNode.DevPath) != sizeof(NVME_NAMESPACE_DEVICE_PATH))) {
 776          return EFI_UNSUPPORTED;
 777       }
 778     }
 779   }
 780
 781   //
 782   // Open the EFI Device Path protocol needed to perform the supported test
 783   //
 784   Status = gBS->OpenProtocol (
 785                   Controller,
 786                   &gEfiDevicePathProtocolGuid,
 787                   (VOID **) &ParentDevicePath,
 788                   This->DriverBindingHandle,
 789                   Controller,
 790                   EFI_OPEN_PROTOCOL_BY_DRIVER
 791                   );
 792   if (Status == EFI_ALREADY_STARTED) {
 793     return EFI_SUCCESS;
 794   }
 795
 796   if (EFI_ERROR (Status)) {
 797     return Status;
 798   }
 799
 800   //
 801   // Close protocol, don't use device path protocol in the Support() function
 802   //
 803   gBS->CloseProtocol (
 804          Controller,
 805          &gEfiDevicePathProtocolGuid,
 806          This->DriverBindingHandle,
 807          Controller
 808          );
 809
 810   //
 811   // Attempt to Open PCI I/O Protocol
 812   //
 813   Status = gBS->OpenProtocol (
 814                   Controller,
 815                   &gEfiPciIoProtocolGuid,
 816                   (VOID **) &PciIo,
 817                   This->DriverBindingHandle,
 818                   Controller,
 819                   EFI_OPEN_PROTOCOL_BY_DRIVER
 820                   );
 821   if (Status == EFI_ALREADY_STARTED) {
 822     return EFI_SUCCESS;
 823   }
 824
 825   if (EFI_ERROR (Status)) {
 826     return Status;
 827   }
 828
 829   //
 830   // Now further check the PCI header: Base class (offset 0x0B) and Sub Class (offset 0x0A).
 831   // This controller should be a Nvm Express controller.
 832   //
 833   Status = PciIo->Pci.Read (
 834                         PciIo,
 835                         EfiPciIoWidthUint8,
 836                         PCI_CLASSCODE_OFFSET,
 837                         sizeof (ClassCode),
 838                         ClassCode
 839                         );
 840   if (EFI_ERROR (Status)) {
 841     goto Done;
 842   }
 843
 844   //
 845   // Examine Nvm Express controller PCI Configuration table fields
 846   //
 847   if ((ClassCode[0] != PCI_IF_NVMHCI) || (ClassCode[1] != PCI_CLASS_MASS_STORAGE_NVM) || (ClassCode[2] != PCI_CLASS_MASS_STORAGE)) {
 848     Status = EFI_UNSUPPORTED;
 849   }
 850
 851 Done:
 852   gBS->CloseProtocol (
 853          Controller,
 854          &gEfiPciIoProtocolGuid,
 855          This->DriverBindingHandle,
 856          Controller
 857          );
 858
 859   return Status;
 860 }
 861
 862
 863 /**
 864   Starts a device controller or a bus controller.
 865
 866   The Start() function is designed to be invoked from the EFI boot service ConnectController().
 867   As a result, much of the error checking on the parameters to Start() has been moved into this
 868   common boot service. It is legal to call Start() from other locations,
 869   but the following calling restrictions must be followed or the system behavior will not be deterministic.
 870   1. ControllerHandle must be a valid EFI_HANDLE.
 871   2. If RemainingDevicePath is not NULL, then it must be a pointer to a naturally aligned
 872      EFI_DEVICE_PATH_PROTOCOL.
 873   3. Prior to calling Start(), the Supported() function for the driver specified by This must
 874      have been called with the same calling parameters, and Supported() must have returned EFI_SUCCESS.
 875
 876   @param[in]  This                 A pointer to the EFI_DRIVER_BINDING_PROTOCOL instance.
 877   @param[in]  ControllerHandle     The handle of the controller to start. This handle
 878                                    must support a protocol interface that supplies
 879                                    an I/O abstraction to the driver.
 880   @param[in]  RemainingDevicePath  A pointer to the remaining portion of a device path.  This
 881                                    parameter is ignored by device drivers, and is optional for bus
 882                                    drivers. For a bus driver, if this parameter is NULL, then handles
 883                                    for all the children of Controller are created by this driver.
 884                                    If this parameter is not NULL and the first Device Path Node is
 885                                    not the End of Device Path Node, then only the handle for the
 886                                    child device specified by the first Device Path Node of
 887                                    RemainingDevicePath is created by this driver.
 888                                    If the first Device Path Node of RemainingDevicePath is
 889                                    the End of Device Path Node, no child handle is created by this
 890                                    driver.
 891
 892   @retval EFI_SUCCESS              The device was started.
 893   @retval EFI_DEVICE_ERROR         The device could not be started due to a device error.Currently not implemented.
 894   @retval EFI_OUT_OF_RESOURCES     The request could not be completed due to a lack of resources.
 895   @retval Others                   The driver failded to start the device.
 896
 897 **/
 898 EFI_STATUS
 899 EFIAPI
 900 NvmExpressDriverBindingStart (
 901   IN EFI_DRIVER_BINDING_PROTOCOL  *This,
 902   IN EFI_HANDLE                   Controller,
 903   IN EFI_DEVICE_PATH_PROTOCOL     *RemainingDevicePath
 904   )
 905 {
 906   EFI_STATUS                          Status;
 907   EFI_PCI_IO_PROTOCOL                 *PciIo;
 908   NVME_CONTROLLER_PRIVATE_DATA        *Private;
 909   EFI_DEVICE_PATH_PROTOCOL            *ParentDevicePath;
 910   UINT32                              NamespaceId;
 911   EFI_PHYSICAL_ADDRESS                MappedAddr;
 912   UINTN                               Bytes;
 913   EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL  *Passthru;
 914
 915   DEBUG ((EFI_D_INFO, "NvmExpressDriverBindingStart: start\n"));
 916
 917   Private          = NULL;
 918   Passthru         = NULL;
 919   ParentDevicePath = NULL;
 920
 921   Status = gBS->OpenProtocol (
 922                   Controller,
 923                   &gEfiDevicePathProtocolGuid,
 924                   (VOID **) &ParentDevicePath,
 925                   This->DriverBindingHandle,
 926                   Controller,
 927                   EFI_OPEN_PROTOCOL_BY_DRIVER
 928                   );
 929   if ((EFI_ERROR (Status)) && (Status != EFI_ALREADY_STARTED)) {
 930     return Status;
 931   }
 932
 933   Status = gBS->OpenProtocol (
 934                   Controller,
 935                   &gEfiPciIoProtocolGuid,
 936                   (VOID **) &PciIo,
 937                   This->DriverBindingHandle,
 938                   Controller,
 939                   EFI_OPEN_PROTOCOL_BY_DRIVER
 940                   );
 941
 942   if (EFI_ERROR (Status) && (Status != EFI_ALREADY_STARTED)) {
 943     return Status;
 944   }
 945
 946   //
 947   // Check EFI_ALREADY_STARTED to reuse the original NVME_CONTROLLER_PRIVATE_DATA.
 948   //
 949   if (Status != EFI_ALREADY_STARTED) {
 950     Private = AllocateZeroPool (sizeof (NVME_CONTROLLER_PRIVATE_DATA));
 951
 952     if (Private == NULL) {
 953       DEBUG ((EFI_D_ERROR, "NvmExpressDriverBindingStart: allocating pool for Nvme Private Data failed!\n"));
 954       Status = EFI_OUT_OF_RESOURCES;
 955       goto Exit;
 956     }
 957
 958     //
 959     // 6 x 4kB aligned buffers will be carved out of this buffer.
 960     // 1st 4kB boundary is the start of the admin submission queue.
 961     // 2nd 4kB boundary is the start of the admin completion queue.
 962     // 3rd 4kB boundary is the start of I/O submission queue #1.
 963     // 4th 4kB boundary is the start of I/O completion queue #1.
 964     // 5th 4kB boundary is the start of I/O submission queue #2.
 965     // 6th 4kB boundary is the start of I/O completion queue #2.
 966     //
 967     // Allocate 6 pages of memory, then map it for bus master read and write.
 968     //
 969     Status = PciIo->AllocateBuffer (
 970                       PciIo,
 971                       AllocateAnyPages,
 972                       EfiBootServicesData,
 973                       6,
 974                       (VOID**)&Private->Buffer,
 975                       0
 976                       );
 977     if (EFI_ERROR (Status)) {
 978       goto Exit;
 979     }
 980
 981     Bytes = EFI_PAGES_TO_SIZE (6);
 982     Status = PciIo->Map (
 983                       PciIo,
 984                       EfiPciIoOperationBusMasterCommonBuffer,
 985                       Private->Buffer,
 986                       &Bytes,
 987                       &MappedAddr,
 988                       &Private->Mapping
 989                       );
 990
 991     if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (6))) {
 992       goto Exit;
 993     }
 994
 995     Private->BufferPciAddr = (UINT8 *)(UINTN)MappedAddr;
 996
 997     Private->Signature = NVME_CONTROLLER_PRIVATE_DATA_SIGNATURE;
 998     Private->ControllerHandle          = Controller;
 999     Private->ImageHandle               = This->DriverBindingHandle;
1000     Private->DriverBindingHandle       = This->DriverBindingHandle;
1001     Private->PciIo                     = PciIo;
1002     Private->ParentDevicePath          = ParentDevicePath;
1003     Private->Passthru.Mode             = &Private->PassThruMode;
1004     Private->Passthru.PassThru         = NvmExpressPassThru;
1005     Private->Passthru.GetNextNamespace = NvmExpressGetNextNamespace;
1006     Private->Passthru.BuildDevicePath  = NvmExpressBuildDevicePath;
1007     Private->Passthru.GetNamespace     = NvmExpressGetNamespace;
1008     CopyMem (&Private->PassThruMode, &gEfiNvmExpressPassThruMode, sizeof (EFI_NVM_EXPRESS_PASS_THRU_MODE));
1009     InitializeListHead (&Private->AsyncPassThruQueue);
1010     InitializeListHead (&Private->UnsubmittedSubtasks);
1011
1012     Status = NvmeControllerInit (Private);
1013     if (EFI_ERROR(Status)) {
1014       goto Exit;
1015     }
1016
1017     //
1018     // Start the asynchronous I/O completion monitor
1019     //
1020     Status = gBS->CreateEvent (
1021                     EVT_TIMER | EVT_NOTIFY_SIGNAL,
1022                     TPL_NOTIFY,
1023                     ProcessAsyncTaskList,
1024                     Private,
1025                     &Private->TimerEvent
1026                     );
1027     if (EFI_ERROR (Status)) {
1028       goto Exit;
1029     }
1030
1031     Status = gBS->SetTimer (
1032                     Private->TimerEvent,
1033                     TimerPeriodic,
1034                     NVME_HC_ASYNC_TIMER
1035                     );
1036     if (EFI_ERROR (Status)) {
1037       goto Exit;
1038     }
1039
1040     Status = gBS->InstallMultipleProtocolInterfaces (
1041                     &Controller,
1042                     &gEfiNvmExpressPassThruProtocolGuid,
1043                     &Private->Passthru,
1044                     NULL
1045                     );
1046     if (EFI_ERROR (Status)) {
1047       goto Exit;
1048     }
1049
1050     NvmeRegisterShutdownNotification ();
1051   } else {
1052     Status = gBS->OpenProtocol (
1053                     Controller,
1054                     &gEfiNvmExpressPassThruProtocolGuid,
1055                     (VOID **) &Passthru,
1056                     This->DriverBindingHandle,
1057                     Controller,
1058                     EFI_OPEN_PROTOCOL_GET_PROTOCOL
1059                     );
1060     if (EFI_ERROR (Status)) {
1061       goto Exit;
1062     }
1063
1064     Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (Passthru);
1065   }
1066
1067   if (RemainingDevicePath == NULL) {
1068     //
1069     // Enumerate all NVME namespaces in the controller
1070     //
1071     Status = DiscoverAllNamespaces (
1072                Private
1073                );
1074
1075   } else if (!IsDevicePathEnd (RemainingDevicePath)) {
1076     //
1077     // Enumerate the specified NVME namespace
1078     //
1079     Status = Private->Passthru.GetNamespace (
1080                                  &Private->Passthru,
1081                                  RemainingDevicePath,
1082                                  &NamespaceId
1083                                  );
1084
1085     if (!EFI_ERROR (Status)) {
1086       Status = EnumerateNvmeDevNamespace (
1087                  Private,
1088                  NamespaceId
1089                  );
1090     }
1091   }
1092
1093   DEBUG ((EFI_D_INFO, "NvmExpressDriverBindingStart: end successfully\n"));
1094   return EFI_SUCCESS;
1095
1096 Exit:
1097   if ((Private != NULL) && (Private->Mapping != NULL)) {
1098     PciIo->Unmap (PciIo, Private->Mapping);
1099   }
1100
1101   if ((Private != NULL) && (Private->Buffer != NULL)) {
1102     PciIo->FreeBuffer (PciIo, 6, Private->Buffer);
1103   }
1104
1105   if ((Private != NULL) && (Private->ControllerData != NULL)) {
1106     FreePool (Private->ControllerData);
1107   }
1108
1109   if (Private != NULL) {
1110     if (Private->TimerEvent != NULL) {
1111       gBS->CloseEvent (Private->TimerEvent);
1112     }
1113
1114     FreePool (Private);
1115   }
1116
1117   gBS->CloseProtocol (
1118          Controller,
1119          &gEfiPciIoProtocolGuid,
1120          This->DriverBindingHandle,
1121          Controller
1122          );
1123
1124   gBS->CloseProtocol (
1125          Controller,
1126          &gEfiDevicePathProtocolGuid,
1127          This->DriverBindingHandle,
1128          Controller
1129          );
1130
1131   DEBUG ((EFI_D_INFO, "NvmExpressDriverBindingStart: end with %r\n", Status));
1132
1133   return Status;
1134 }
1135
1136
1137 /**
1138   Stops a device controller or a bus controller.
1139
1140   The Stop() function is designed to be invoked from the EFI boot service DisconnectController().
1141   As a result, much of the error checking on the parameters to Stop() has been moved
1142   into this common boot service. It is legal to call Stop() from other locations,
1143   but the following calling restrictions must be followed or the system behavior will not be deterministic.
1144   1. ControllerHandle must be a valid EFI_HANDLE that was used on a previous call to this
1145      same driver's Start() function.
1146   2. The first NumberOfChildren handles of ChildHandleBuffer must all be a valid
1147      EFI_HANDLE. In addition, all of these handles must have been created in this driver's
1148      Start() function, and the Start() function must have called OpenProtocol() on
1149      ControllerHandle with an Attribute of EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER.
1150
1151   @param[in]  This              A pointer to the EFI_DRIVER_BINDING_PROTOCOL instance.
1152   @param[in]  ControllerHandle  A handle to the device being stopped. The handle must
1153                                 support a bus specific I/O protocol for the driver
1154                                 to use to stop the device.
1155   @param[in]  NumberOfChildren  The number of child device handles in ChildHandleBuffer.
1156   @param[in]  ChildHandleBuffer An array of child handles to be freed. May be NULL
1157                                 if NumberOfChildren is 0.
1158
1159   @retval EFI_SUCCESS           The device was stopped.
1160   @retval EFI_DEVICE_ERROR      The device could not be stopped due to a device error.
1161
1162 **/
1163 EFI_STATUS
1164 EFIAPI
1165 NvmExpressDriverBindingStop (
1166   IN  EFI_DRIVER_BINDING_PROTOCOL     *This,
1167   IN  EFI_HANDLE                      Controller,
1168   IN  UINTN                           NumberOfChildren,
1169   IN  EFI_HANDLE                      *ChildHandleBuffer
1170   )
1171 {
1172   EFI_STATUS                          Status;
1173   BOOLEAN                             AllChildrenStopped;
1174   UINTN                               Index;
1175   NVME_CONTROLLER_PRIVATE_DATA        *Private;
1176   EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL  *PassThru;
1177   BOOLEAN                             IsEmpty;
1178   EFI_TPL                             OldTpl;
1179
1180   if (NumberOfChildren == 0) {
1181     Status = gBS->OpenProtocol (
1182                     Controller,
1183                     &gEfiNvmExpressPassThruProtocolGuid,
1184                     (VOID **) &PassThru,
1185                     This->DriverBindingHandle,
1186                     Controller,
1187                     EFI_OPEN_PROTOCOL_GET_PROTOCOL
1188                     );
1189
1190     if (!EFI_ERROR (Status)) {
1191       Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (PassThru);
1192
1193       //
1194       // Wait for the asynchronous PassThru queue to become empty.
1195       //
1196       while (TRUE) {
1197         OldTpl  = gBS->RaiseTPL (TPL_NOTIFY);
1198         IsEmpty = IsListEmpty (&Private->AsyncPassThruQueue) &&
1199                   IsListEmpty (&Private->UnsubmittedSubtasks);
1200         gBS->RestoreTPL (OldTpl);
1201
1202         if (IsEmpty) {
1203           break;
1204         }
1205
1206         gBS->Stall (100);
1207       }
1208
1209       gBS->UninstallMultipleProtocolInterfaces (
1210             Controller,
1211             &gEfiNvmExpressPassThruProtocolGuid,
1212             PassThru,
1213             NULL
1214             );
1215
1216       if (Private->TimerEvent != NULL) {
1217         gBS->CloseEvent (Private->TimerEvent);
1218       }
1219
1220       if (Private->Mapping != NULL) {
1221         Private->PciIo->Unmap (Private->PciIo, Private->Mapping);
1222       }
1223
1224       if (Private->Buffer != NULL) {
1225         Private->PciIo->FreeBuffer (Private->PciIo, 6, Private->Buffer);
1226       }
1227
1228       FreePool (Private->ControllerData);
1229       FreePool (Private);
1230     }
1231
1232     gBS->CloseProtocol (
1233           Controller,
1234           &gEfiPciIoProtocolGuid,
1235           This->DriverBindingHandle,
1236           Controller
1237           );
1238     gBS->CloseProtocol (
1239           Controller,
1240           &gEfiDevicePathProtocolGuid,
1241           This->DriverBindingHandle,
1242           Controller
1243           );
1244
1245     NvmeUnregisterShutdownNotification ();
1246
1247     return EFI_SUCCESS;
1248   }
1249
1250   AllChildrenStopped = TRUE;
1251
1252   for (Index = 0; Index < NumberOfChildren; Index++) {
1253     Status = UnregisterNvmeNamespace (This, Controller, ChildHandleBuffer[Index]);
1254     if (EFI_ERROR (Status)) {
1255       AllChildrenStopped = FALSE;
1256     }
1257   }
1258
1259   if (!AllChildrenStopped) {
1260     return EFI_DEVICE_ERROR;
1261   }
1262
1263   return EFI_SUCCESS;
1264 }
1265
1266 /**
1267   This is the unload handle for the NVM Express driver.
1268
1269   Disconnect the driver specified by ImageHandle from the NVMe device in the handle database.
1270   Uninstall all the protocols installed in the driver.
1271
1272   @param[in]  ImageHandle       The drivers' driver image.
1273
1274   @retval EFI_SUCCESS           The image is unloaded.
1275   @retval Others                Failed to unload the image.
1276
1277 **/
1278 EFI_STATUS
1279 EFIAPI
1280 NvmExpressUnload (
1281   IN EFI_HANDLE             ImageHandle
1282   )
1283 {
1284   EFI_STATUS                        Status;
1285   EFI_HANDLE                        *DeviceHandleBuffer;
1286   UINTN                             DeviceHandleCount;
1287   UINTN                             Index;
1288   EFI_COMPONENT_NAME_PROTOCOL       *ComponentName;
1289   EFI_COMPONENT_NAME2_PROTOCOL      *ComponentName2;
1290
1291   //
1292   // Get the list of the device handles managed by this driver.
1293   // If there is an error getting the list, then means the driver
1294   // doesn't manage any device. At this way, we would only close
1295   // those protocols installed at image handle.
1296   //
1297   DeviceHandleBuffer = NULL;
1298   Status = gBS->LocateHandleBuffer (
1299                   ByProtocol,
1300                   &gEfiNvmExpressPassThruProtocolGuid,
1301                   NULL,
1302                   &DeviceHandleCount,
1303                   &DeviceHandleBuffer
1304                   );
1305
1306   if (!EFI_ERROR (Status)) {
1307     //
1308     // Disconnect the driver specified by ImageHandle from all
1309     // the devices in the handle database.
1310     //
1311     for (Index = 0; Index < DeviceHandleCount; Index++) {
1312       Status = gBS->DisconnectController (
1313                       DeviceHandleBuffer[Index],
1314                       ImageHandle,
1315                       NULL
1316                       );
1317       if (EFI_ERROR (Status)) {
1318         goto EXIT;
1319       }
1320     }
1321   }
1322
1323   //
1324   // Uninstall all the protocols installed in the driver entry point
1325   //
1326   Status = gBS->UninstallMultipleProtocolInterfaces (
1327                   ImageHandle,
1328                   &gEfiDriverBindingProtocolGuid,
1329                   &gNvmExpressDriverBinding,
1330                   &gEfiDriverSupportedEfiVersionProtocolGuid,
1331                   &gNvmExpressDriverSupportedEfiVersion,
1332                   NULL
1333                   );
1334
1335   if (EFI_ERROR (Status)) {
1336     goto EXIT;
1337   }
1338
1339   //
1340   // Note we have to one by one uninstall the following protocols.
1341   // It's because some of them are optionally installed based on
1342   // the following PCD settings.
1343   //   gEfiMdePkgTokenSpaceGuid.PcdDriverDiagnosticsDisable
1344   //   gEfiMdePkgTokenSpaceGuid.PcdComponentNameDisable
1345   //   gEfiMdePkgTokenSpaceGuid.PcdDriverDiagnostics2Disable
1346   //   gEfiMdePkgTokenSpaceGuid.PcdComponentName2Disable
1347   //
1348   Status = gBS->HandleProtocol (
1349                   ImageHandle,
1350                   &gEfiComponentNameProtocolGuid,
1351                   (VOID **) &ComponentName
1352                   );
1353   if (!EFI_ERROR (Status)) {
1354     gBS->UninstallProtocolInterface (
1355            ImageHandle,
1356            &gEfiComponentNameProtocolGuid,
1357            ComponentName
1358            );
1359   }
1360
1361   Status = gBS->HandleProtocol (
1362                   ImageHandle,
1363                   &gEfiComponentName2ProtocolGuid,
1364                   (VOID **) &ComponentName2
1365                   );
1366   if (!EFI_ERROR (Status)) {
1367     gBS->UninstallProtocolInterface (
1368            ImageHandle,
1369            &gEfiComponentName2ProtocolGuid,
1370            ComponentName2
1371            );
1372   }
1373
1374   Status = EFI_SUCCESS;
1375
1376 EXIT:
1377   //
1378   // Free the buffer containing the list of handles from the handle database
1379   //
1380   if (DeviceHandleBuffer != NULL) {
1381     gBS->FreePool (DeviceHandleBuffer);
1382   }
1383   return Status;
1384 }
1385
1386 /**
1387   The entry point for Nvm Express driver, used to install Nvm Express driver on the ImageHandle.
1388
1389   @param  ImageHandle   The firmware allocated handle for this driver image.
1390   @param  SystemTable   Pointer to the EFI system table.
1391
1392   @retval EFI_SUCCESS   Driver loaded.
1393   @retval other         Driver not loaded.
1394
1395 **/
1396 EFI_STATUS
1397 EFIAPI
1398 NvmExpressDriverEntry (
1399   IN EFI_HANDLE        ImageHandle,
1400   IN EFI_SYSTEM_TABLE  *SystemTable
1401   )
1402 {
1403   EFI_STATUS              Status;
1404
1405   Status = EfiLibInstallDriverBindingComponentName2 (
1406              ImageHandle,
1407              SystemTable,
1408              &gNvmExpressDriverBinding,
1409              ImageHandle,
1410              &gNvmExpressComponentName,
1411              &gNvmExpressComponentName2
1412              );
1413   ASSERT_EFI_ERROR (Status);
1414
1415   //
1416   // Install EFI Driver Supported EFI Version Protocol required for
1417   // EFI drivers that are on PCI and other plug in cards.
1418   //
1419   gNvmExpressDriverSupportedEfiVersion.FirmwareVersion = 0x00020028;
1420   Status = gBS->InstallMultipleProtocolInterfaces (
1421                   &ImageHandle,
1422                   &gEfiDriverSupportedEfiVersionProtocolGuid,
1423                   &gNvmExpressDriverSupportedEfiVersion,
1424                   NULL
1425                   );
1426   ASSERT_EFI_ERROR (Status);
1427   return Status;
1428 }