MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.c

   1 /** @file
   2   NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows
   3   NVM Express specification.
   4
   5   Copyright (c) 2013 - 2016, Intel Corporation. All rights reserved.<BR>
   6   This program and the accompanying materials
   7   are licensed and made available under the terms and conditions of the BSD License
   8   which accompanies this distribution.  The full text of the license may be found at
   9   http://opensource.org/licenses/bsd-license.php.
  10
  11   THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  12   WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  13
  14 **/
  15
  16 #include "NvmExpress.h"
  17
  18 //
  19 // NVM Express Driver Binding Protocol Instance
  20 //
  21 EFI_DRIVER_BINDING_PROTOCOL gNvmExpressDriverBinding = {
  22   NvmExpressDriverBindingSupported,
  23   NvmExpressDriverBindingStart,
  24   NvmExpressDriverBindingStop,
  25   0x10,
  26   NULL,
  27   NULL
  28 };
  29
  30 //
  31 // NVM Express EFI Driver Supported EFI Version Protocol Instance
  32 //
  33 EFI_DRIVER_SUPPORTED_EFI_VERSION_PROTOCOL gNvmExpressDriverSupportedEfiVersion = {
  34   sizeof (EFI_DRIVER_SUPPORTED_EFI_VERSION_PROTOCOL), // Size of Protocol structure.
  35   0                                                   // Version number to be filled at start up.
  36 };
  37
  38 //
  39 // Template for NVM Express Pass Thru Mode data structure.
  40 //
  41 GLOBAL_REMOVE_IF_UNREFERENCED EFI_NVM_EXPRESS_PASS_THRU_MODE gEfiNvmExpressPassThruMode = {
  42   EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL   |
  43   EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL    |
  44   EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_NONBLOCKIO |
  45   EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_CMD_SET_NVM,
  46   sizeof (UINTN),
  47   0x10100
  48 };
  49
  50 /**
  51   Check if the specified Nvm Express device namespace is active, and create child handles
  52   for them with BlockIo and DiskInfo protocol instances.
  53
  54   @param[in] Private         The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
  55   @param[in] NamespaceId     The NVM Express namespace ID  for which a device path node is to be
  56                              allocated and built. Caller must set the NamespaceId to zero if the
  57                              device path node will contain a valid UUID.
  58
  59   @retval EFI_SUCCESS        All the namespaces in the device are successfully enumerated.
  60   @return Others             Some error occurs when enumerating the namespaces.
  61
  62 **/
  63 EFI_STATUS
  64 EnumerateNvmeDevNamespace (
  65   IN NVME_CONTROLLER_PRIVATE_DATA       *Private,
  66   UINT32                                NamespaceId
  67   )
  68 {
  69   NVME_ADMIN_NAMESPACE_DATA             *NamespaceData;
  70   EFI_DEVICE_PATH_PROTOCOL              *NewDevicePathNode;
  71   EFI_DEVICE_PATH_PROTOCOL              *DevicePath;
  72   EFI_HANDLE                            DeviceHandle;
  73   EFI_DEVICE_PATH_PROTOCOL              *ParentDevicePath;
  74   EFI_DEVICE_PATH_PROTOCOL              *RemainingDevicePath;
  75   NVME_DEVICE_PRIVATE_DATA              *Device;
  76   EFI_STATUS                            Status;
  77   UINT32                                Lbads;
  78   UINT32                                Flbas;
  79   UINT32                                LbaFmtIdx;
  80   UINT8                                 Sn[21];
  81   UINT8                                 Mn[41];
  82   VOID                                  *DummyInterface;
  83
  84   NewDevicePathNode = NULL;
  85   DevicePath        = NULL;
  86   Device            = NULL;
  87
  88   //
  89   // Allocate a buffer for Identify Namespace data
  90   //
  91   NamespaceData = AllocateZeroPool(sizeof (NVME_ADMIN_NAMESPACE_DATA));
  92   if(NamespaceData == NULL) {
  93     return EFI_OUT_OF_RESOURCES;
  94   }
  95
  96   ParentDevicePath = Private->ParentDevicePath;
  97   //
  98   // Identify Namespace
  99   //
 100   Status = NvmeIdentifyNamespace (
 101              Private,
 102              NamespaceId,
 103              (VOID *)NamespaceData
 104              );
 105   if (EFI_ERROR(Status)) {
 106     goto Exit;
 107   }
 108   //
 109   // Validate Namespace
 110   //
 111   if (NamespaceData->Ncap == 0) {
 112     Status = EFI_DEVICE_ERROR;
 113   } else {
 114     //
 115     // allocate device private data for each discovered namespace
 116     //
 117     Device = AllocateZeroPool(sizeof(NVME_DEVICE_PRIVATE_DATA));
 118     if (Device == NULL) {
 119       Status = EFI_OUT_OF_RESOURCES;
 120       goto Exit;
 121     }
 122
 123     //
 124     // Initialize SSD namespace instance data
 125     //
 126     Device->Signature           = NVME_DEVICE_PRIVATE_DATA_SIGNATURE;
 127     Device->NamespaceId         = NamespaceId;
 128     Device->NamespaceUuid       = NamespaceData->Eui64;
 129
 130     Device->ControllerHandle    = Private->ControllerHandle;
 131     Device->DriverBindingHandle = Private->DriverBindingHandle;
 132     Device->Controller          = Private;
 133
 134     //
 135     // Build BlockIo media structure
 136     //
 137     Device->Media.MediaId        = 0;
 138     Device->Media.RemovableMedia = FALSE;
 139     Device->Media.MediaPresent   = TRUE;
 140     Device->Media.LogicalPartition = FALSE;
 141     Device->Media.ReadOnly       = FALSE;
 142     Device->Media.WriteCaching   = FALSE;
 143     Device->Media.IoAlign        = Private->PassThruMode.IoAlign;
 144
 145     Flbas     = NamespaceData->Flbas;
 146     LbaFmtIdx = Flbas & 0xF;
 147     Lbads     = NamespaceData->LbaFormat[LbaFmtIdx].Lbads;
 148     Device->Media.BlockSize = (UINT32)1 << Lbads;
 149
 150     Device->Media.LastBlock                     = NamespaceData->Nsze - 1;
 151     Device->Media.LogicalBlocksPerPhysicalBlock = 1;
 152     Device->Media.LowestAlignedLba              = 1;
 153
 154     //
 155     // Create BlockIo Protocol instance
 156     //
 157     Device->BlockIo.Revision     = EFI_BLOCK_IO_PROTOCOL_REVISION2;
 158     Device->BlockIo.Media        = &Device->Media;
 159     Device->BlockIo.Reset        = NvmeBlockIoReset;
 160     Device->BlockIo.ReadBlocks   = NvmeBlockIoReadBlocks;
 161     Device->BlockIo.WriteBlocks  = NvmeBlockIoWriteBlocks;
 162     Device->BlockIo.FlushBlocks  = NvmeBlockIoFlushBlocks;
 163
 164     //
 165     // Create BlockIo2 Protocol instance
 166     //
 167     Device->BlockIo2.Media          = &Device->Media;
 168     Device->BlockIo2.Reset          = NvmeBlockIoResetEx;
 169     Device->BlockIo2.ReadBlocksEx   = NvmeBlockIoReadBlocksEx;
 170     Device->BlockIo2.WriteBlocksEx  = NvmeBlockIoWriteBlocksEx;
 171     Device->BlockIo2.FlushBlocksEx  = NvmeBlockIoFlushBlocksEx;
 172     InitializeListHead (&Device->AsyncQueue);
 173
 174     //
 175     // Create StorageSecurityProtocol Instance
 176     //
 177     Device->StorageSecurity.ReceiveData = NvmeStorageSecurityReceiveData;
 178     Device->StorageSecurity.SendData    = NvmeStorageSecuritySendData;
 179
 180     //
 181     // Create DiskInfo Protocol instance
 182     //
 183     CopyMem (&Device->NamespaceData, NamespaceData, sizeof (NVME_ADMIN_NAMESPACE_DATA));
 184     InitializeDiskInfo (Device);
 185
 186     //
 187     // Create a Nvm Express Namespace Device Path Node
 188     //
 189     Status = Private->Passthru.BuildDevicePath (
 190                                  &Private->Passthru,
 191                                  Device->NamespaceId,
 192                                  &NewDevicePathNode
 193                                  );
 194
 195     if (EFI_ERROR(Status)) {
 196       goto Exit;
 197     }
 198
 199     //
 200     // Append the SSD node to the controller's device path
 201     //
 202     DevicePath = AppendDevicePathNode (ParentDevicePath, NewDevicePathNode);
 203     if (DevicePath == NULL) {
 204       Status = EFI_OUT_OF_RESOURCES;
 205       goto Exit;
 206     }
 207
 208     DeviceHandle = NULL;
 209     RemainingDevicePath = DevicePath;
 210     Status = gBS->LocateDevicePath (&gEfiDevicePathProtocolGuid, &RemainingDevicePath, &DeviceHandle);
 211     if (!EFI_ERROR (Status) && (DeviceHandle != NULL) && IsDevicePathEnd(RemainingDevicePath)) {
 212       Status = EFI_ALREADY_STARTED;
 213       FreePool (DevicePath);
 214       goto Exit;
 215     }
 216
 217     Device->DevicePath = DevicePath;
 218
 219     //
 220     // Make sure the handle is NULL so we create a new handle
 221     //
 222     Device->DeviceHandle = NULL;
 223
 224     Status = gBS->InstallMultipleProtocolInterfaces (
 225                     &Device->DeviceHandle,
 226                     &gEfiDevicePathProtocolGuid,
 227                     Device->DevicePath,
 228                     &gEfiBlockIoProtocolGuid,
 229                     &Device->BlockIo,
 230                     &gEfiBlockIo2ProtocolGuid,
 231                     &Device->BlockIo2,
 232                     &gEfiDiskInfoProtocolGuid,
 233                     &Device->DiskInfo,
 234                     NULL
 235                     );
 236
 237     if(EFI_ERROR(Status)) {
 238       goto Exit;
 239     }
 240
 241     //
 242     // Check if the NVMe controller supports the Security Send and Security Receive commands
 243     //
 244     if ((Private->ControllerData->Oacs & SECURITY_SEND_RECEIVE_SUPPORTED) != 0) {
 245       Status = gBS->InstallProtocolInterface (
 246                       &Device->DeviceHandle,
 247                       &gEfiStorageSecurityCommandProtocolGuid,
 248                       EFI_NATIVE_INTERFACE,
 249                       &Device->StorageSecurity
 250                       );
 251       if(EFI_ERROR(Status)) {
 252         gBS->UninstallMultipleProtocolInterfaces (
 253                &Device->DeviceHandle,
 254                &gEfiDevicePathProtocolGuid,
 255                Device->DevicePath,
 256                &gEfiBlockIoProtocolGuid,
 257                &Device->BlockIo,
 258                &gEfiBlockIo2ProtocolGuid,
 259                &Device->BlockIo2,
 260                &gEfiDiskInfoProtocolGuid,
 261                &Device->DiskInfo,
 262                NULL
 263                );
 264         goto Exit;
 265       }
 266     }
 267
 268     gBS->OpenProtocol (
 269            Private->ControllerHandle,
 270            &gEfiNvmExpressPassThruProtocolGuid,
 271            (VOID **) &DummyInterface,
 272            Private->DriverBindingHandle,
 273            Device->DeviceHandle,
 274            EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER
 275            );
 276
 277     //
 278     // Dump NvmExpress Identify Namespace Data
 279     //
 280     DEBUG ((EFI_D_INFO, " == NVME IDENTIFY NAMESPACE [%d] DATA ==\n", NamespaceId));
 281     DEBUG ((EFI_D_INFO, "    NSZE        : 0x%x\n", NamespaceData->Nsze));
 282     DEBUG ((EFI_D_INFO, "    NCAP        : 0x%x\n", NamespaceData->Ncap));
 283     DEBUG ((EFI_D_INFO, "    NUSE        : 0x%x\n", NamespaceData->Nuse));
 284     DEBUG ((EFI_D_INFO, "    LBAF0.LBADS : 0x%x\n", (NamespaceData->LbaFormat[0].Lbads)));
 285
 286     //
 287     // Build controller name for Component Name (2) protocol.
 288     //
 289     CopyMem (Sn, Private->ControllerData->Sn, sizeof (Private->ControllerData->Sn));
 290     Sn[20] = 0;
 291     CopyMem (Mn, Private->ControllerData->Mn, sizeof (Private->ControllerData->Mn));
 292     Mn[40] = 0;
 293     UnicodeSPrintAsciiFormat (Device->ModelName, sizeof (Device->ModelName), "%a-%a-%x", Sn, Mn, NamespaceData->Eui64);
 294
 295     AddUnicodeString2 (
 296       "eng",
 297       gNvmExpressComponentName.SupportedLanguages,
 298       &Device->ControllerNameTable,
 299       Device->ModelName,
 300       TRUE
 301       );
 302
 303     AddUnicodeString2 (
 304       "en",
 305       gNvmExpressComponentName2.SupportedLanguages,
 306       &Device->ControllerNameTable,
 307       Device->ModelName,
 308       FALSE
 309       );
 310   }
 311
 312 Exit:
 313   if(NamespaceData != NULL) {
 314     FreePool (NamespaceData);
 315   }
 316
 317   if (NewDevicePathNode != NULL) {
 318     FreePool (NewDevicePathNode);
 319   }
 320
 321   if(EFI_ERROR(Status) && (Device != NULL) && (Device->DevicePath != NULL)) {
 322     FreePool (Device->DevicePath);
 323   }
 324   if(EFI_ERROR(Status) && (Device != NULL)) {
 325     FreePool (Device);
 326   }
 327   return Status;
 328 }
 329
 330 /**
 331   Discover all Nvm Express device namespaces, and create child handles for them with BlockIo
 332   and DiskInfo protocol instances.
 333
 334   @param[in] Private         The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 335
 336   @retval EFI_SUCCESS        All the namespaces in the device are successfully enumerated.
 337   @return Others             Some error occurs when enumerating the namespaces.
 338
 339 **/
 340 EFI_STATUS
 341 DiscoverAllNamespaces (
 342   IN NVME_CONTROLLER_PRIVATE_DATA       *Private
 343   )
 344 {
 345   EFI_STATUS                            Status;
 346   UINT32                                NamespaceId;
 347   EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL    *Passthru;
 348
 349   NamespaceId   = 0xFFFFFFFF;
 350   Passthru      = &Private->Passthru;
 351
 352   while (TRUE) {
 353     Status = Passthru->GetNextNamespace (
 354                          Passthru,
 355                          (UINT32 *)&NamespaceId
 356                          );
 357
 358     if (EFI_ERROR (Status)) {
 359       break;
 360     }
 361
 362     Status = EnumerateNvmeDevNamespace (
 363                Private,
 364                NamespaceId
 365                );
 366
 367     if (EFI_ERROR(Status)) {
 368       continue;
 369     }
 370   }
 371
 372   return EFI_SUCCESS;
 373 }
 374
 375 /**
 376   Unregisters a Nvm Express device namespace.
 377
 378   This function removes the protocols installed on the controller handle and
 379   frees the resources allocated for the namespace.
 380
 381   @param  This                  The pointer to EFI_DRIVER_BINDING_PROTOCOL instance.
 382   @param  Controller            The controller handle of the namespace.
 383   @param  Handle                The child handle.
 384
 385   @retval EFI_SUCCESS           The namespace is successfully unregistered.
 386   @return Others                Some error occurs when unregistering the namespace.
 387
 388 **/
 389 EFI_STATUS
 390 UnregisterNvmeNamespace (
 391   IN  EFI_DRIVER_BINDING_PROTOCOL    *This,
 392   IN  EFI_HANDLE                     Controller,
 393   IN  EFI_HANDLE                     Handle
 394   )
 395 {
 396   EFI_STATUS                               Status;
 397   EFI_BLOCK_IO_PROTOCOL                    *BlockIo;
 398   NVME_DEVICE_PRIVATE_DATA                 *Device;
 399   EFI_STORAGE_SECURITY_COMMAND_PROTOCOL    *StorageSecurity;
 400   BOOLEAN                                  IsEmpty;
 401   EFI_TPL                                  OldTpl;
 402   VOID                                     *DummyInterface;
 403
 404   BlockIo = NULL;
 405
 406   Status = gBS->OpenProtocol (
 407                   Handle,
 408                   &gEfiBlockIoProtocolGuid,
 409                   (VOID **) &BlockIo,
 410                   This->DriverBindingHandle,
 411                   Controller,
 412                   EFI_OPEN_PROTOCOL_GET_PROTOCOL
 413                   );
 414   if (EFI_ERROR (Status)) {
 415     return Status;
 416   }
 417
 418   Device  = NVME_DEVICE_PRIVATE_DATA_FROM_BLOCK_IO (BlockIo);
 419
 420   //
 421   // Wait for the device's asynchronous I/O queue to become empty.
 422   //
 423   while (TRUE) {
 424     OldTpl  = gBS->RaiseTPL (TPL_NOTIFY);
 425     IsEmpty = IsListEmpty (&Device->AsyncQueue);
 426     gBS->RestoreTPL (OldTpl);
 427
 428     if (IsEmpty) {
 429       break;
 430     }
 431
 432     gBS->Stall (100);
 433   }
 434
 435   //
 436   // Close the child handle
 437   //
 438   gBS->CloseProtocol (
 439          Controller,
 440          &gEfiNvmExpressPassThruProtocolGuid,
 441          This->DriverBindingHandle,
 442          Handle
 443          );
 444
 445   //
 446   // The Nvm Express driver installs the BlockIo and DiskInfo in the DriverBindingStart().
 447   // Here should uninstall both of them.
 448   //
 449   Status = gBS->UninstallMultipleProtocolInterfaces (
 450                   Handle,
 451                   &gEfiDevicePathProtocolGuid,
 452                   Device->DevicePath,
 453                   &gEfiBlockIoProtocolGuid,
 454                   &Device->BlockIo,
 455                   &gEfiBlockIo2ProtocolGuid,
 456                   &Device->BlockIo2,
 457                   &gEfiDiskInfoProtocolGuid,
 458                   &Device->DiskInfo,
 459                   NULL
 460                   );
 461
 462   if (EFI_ERROR (Status)) {
 463     gBS->OpenProtocol (
 464            Controller,
 465            &gEfiNvmExpressPassThruProtocolGuid,
 466            (VOID **) &DummyInterface,
 467            This->DriverBindingHandle,
 468            Handle,
 469            EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER
 470            );
 471     return Status;
 472   }
 473
 474   //
 475   // If Storage Security Command Protocol is installed, then uninstall this protocol.
 476   //
 477   Status = gBS->OpenProtocol (
 478                   Handle,
 479                   &gEfiStorageSecurityCommandProtocolGuid,
 480                   (VOID **) &StorageSecurity,
 481                   This->DriverBindingHandle,
 482                   Controller,
 483                   EFI_OPEN_PROTOCOL_GET_PROTOCOL
 484                   );
 485
 486   if (!EFI_ERROR (Status)) {
 487     Status = gBS->UninstallProtocolInterface (
 488                     Handle,
 489                     &gEfiStorageSecurityCommandProtocolGuid,
 490                     &Device->StorageSecurity
 491                     );
 492     if (EFI_ERROR (Status)) {
 493       gBS->OpenProtocol (
 494         Controller,
 495         &gEfiNvmExpressPassThruProtocolGuid,
 496         (VOID **) &DummyInterface,
 497         This->DriverBindingHandle,
 498         Handle,
 499         EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER
 500         );
 501       return Status;
 502     }
 503   }
 504
 505   if(Device->DevicePath != NULL) {
 506     FreePool (Device->DevicePath);
 507   }
 508
 509   if (Device->ControllerNameTable != NULL) {
 510     FreeUnicodeStringTable (Device->ControllerNameTable);
 511   }
 512
 513   FreePool (Device);
 514
 515   return EFI_SUCCESS;
 516 }
 517
 518 /**
 519   Call back function when the timer event is signaled.
 520
 521   @param[in]  Event     The Event this notify function registered to.
 522   @param[in]  Context   Pointer to the context data registered to the
 523                         Event.
 524
 525 **/
 526 VOID
 527 EFIAPI
 528 ProcessAsyncTaskList (
 529   IN EFI_EVENT                    Event,
 530   IN VOID*                        Context
 531   )
 532 {
 533   NVME_CONTROLLER_PRIVATE_DATA         *Private;
 534   EFI_PCI_IO_PROTOCOL                  *PciIo;
 535   NVME_CQ                              *Cq;
 536   UINT16                               QueueId;
 537   UINT32                               Data;
 538   LIST_ENTRY                           *Link;
 539   LIST_ENTRY                           *NextLink;
 540   NVME_PASS_THRU_ASYNC_REQ             *AsyncRequest;
 541   NVME_BLKIO2_SUBTASK                  *Subtask;
 542   NVME_BLKIO2_REQUEST                  *BlkIo2Request;
 543   EFI_BLOCK_IO2_TOKEN                  *Token;
 544   BOOLEAN                              HasNewItem;
 545   EFI_STATUS                           Status;
 546
 547   Private    = (NVME_CONTROLLER_PRIVATE_DATA*)Context;
 548   QueueId    = 2;
 549   Cq         = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
 550   HasNewItem = FALSE;
 551
 552   //
 553   // Submit asynchronous subtasks to the NVMe Submission Queue
 554   //
 555   for (Link = GetFirstNode (&Private->UnsubmittedSubtasks);
 556        !IsNull (&Private->UnsubmittedSubtasks, Link);
 557        Link = NextLink) {
 558     NextLink      = GetNextNode (&Private->UnsubmittedSubtasks, Link);
 559     Subtask       = NVME_BLKIO2_SUBTASK_FROM_LINK (Link);
 560     BlkIo2Request = Subtask->BlockIo2Request;
 561     Token         = BlkIo2Request->Token;
 562     RemoveEntryList (Link);
 563     BlkIo2Request->UnsubmittedSubtaskNum--;
 564
 565     //
 566     // If any previous subtask fails, do not process subsequent ones.
 567     //
 568     if (Token->TransactionStatus != EFI_SUCCESS) {
 569       if (IsListEmpty (&BlkIo2Request->SubtasksQueue) &&
 570           BlkIo2Request->LastSubtaskSubmitted &&
 571           (BlkIo2Request->UnsubmittedSubtaskNum == 0)) {
 572         //
 573         // Remove the BlockIo2 request from the device asynchronous queue.
 574         //
 575         RemoveEntryList (&BlkIo2Request->Link);
 576         FreePool (BlkIo2Request);
 577         gBS->SignalEvent (Token->Event);
 578       }
 579
 580       FreePool (Subtask->CommandPacket->NvmeCmd);
 581       FreePool (Subtask->CommandPacket->NvmeCompletion);
 582       FreePool (Subtask->CommandPacket);
 583       FreePool (Subtask);
 584
 585       continue;
 586     }
 587
 588     Status = Private->Passthru.PassThru (
 589                                  &Private->Passthru,
 590                                  Subtask->NamespaceId,
 591                                  Subtask->CommandPacket,
 592                                  Subtask->Event
 593                                  );
 594     if (Status == EFI_NOT_READY) {
 595       InsertHeadList (&Private->UnsubmittedSubtasks, Link);
 596       BlkIo2Request->UnsubmittedSubtaskNum++;
 597       break;
 598     } else if (EFI_ERROR (Status)) {
 599       Token->TransactionStatus = EFI_DEVICE_ERROR;
 600
 601       if (IsListEmpty (&BlkIo2Request->SubtasksQueue) &&
 602           Subtask->IsLast) {
 603         //
 604         // Remove the BlockIo2 request from the device asynchronous queue.
 605         //
 606         RemoveEntryList (&BlkIo2Request->Link);
 607         FreePool (BlkIo2Request);
 608         gBS->SignalEvent (Token->Event);
 609       }
 610
 611       FreePool (Subtask->CommandPacket->NvmeCmd);
 612       FreePool (Subtask->CommandPacket->NvmeCompletion);
 613       FreePool (Subtask->CommandPacket);
 614       FreePool (Subtask);
 615     } else {
 616       InsertTailList (&BlkIo2Request->SubtasksQueue, Link);
 617       if (Subtask->IsLast) {
 618         BlkIo2Request->LastSubtaskSubmitted = TRUE;
 619       }
 620     }
 621   }
 622
 623   while (Cq->Pt != Private->Pt[QueueId]) {
 624     ASSERT (Cq->Sqid == QueueId);
 625
 626     HasNewItem = TRUE;
 627
 628     //
 629     // Find the command with given Command Id.
 630     //
 631     for (Link = GetFirstNode (&Private->AsyncPassThruQueue);
 632          !IsNull (&Private->AsyncPassThruQueue, Link);
 633          Link = NextLink) {
 634       NextLink = GetNextNode (&Private->AsyncPassThruQueue, Link);
 635       AsyncRequest = NVME_PASS_THRU_ASYNC_REQ_FROM_THIS (Link);
 636       if (AsyncRequest->CommandId == Cq->Cid) {
 637         //
 638         // Copy the Respose Queue entry for this command to the callers
 639         // response buffer.
 640         //
 641         CopyMem (
 642           AsyncRequest->Packet->NvmeCompletion,
 643           Cq,
 644           sizeof(EFI_NVM_EXPRESS_COMPLETION)
 645           );
 646
 647         RemoveEntryList (Link);
 648         gBS->SignalEvent (AsyncRequest->CallerEvent);
 649         FreePool (AsyncRequest);
 650
 651         //
 652         // Update submission queue head.
 653         //
 654         Private->AsyncSqHead = Cq->Sqhd;
 655         break;
 656       }
 657     }
 658
 659     Private->CqHdbl[QueueId].Cqh++;
 660     if (Private->CqHdbl[QueueId].Cqh > NVME_ASYNC_CCQ_SIZE) {
 661       Private->CqHdbl[QueueId].Cqh = 0;
 662       Private->Pt[QueueId] ^= 1;
 663     }
 664
 665     Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
 666   }
 667
 668   if (HasNewItem) {
 669     PciIo = Private->PciIo;
 670     Data  = ReadUnaligned32 ((UINT32*)&Private->CqHdbl[QueueId]);
 671     PciIo->Mem.Write (
 672                  PciIo,
 673                  EfiPciIoWidthUint32,
 674                  NVME_BAR,
 675                  NVME_CQHDBL_OFFSET(QueueId, Private->Cap.Dstrd),
 676                  1,
 677                  &Data
 678                  );
 679   }
 680 }
 681
 682 /**
 683   Tests to see if this driver supports a given controller. If a child device is provided,
 684   it further tests to see if this driver supports creating a handle for the specified child device.
 685
 686   This function checks to see if the driver specified by This supports the device specified by
 687   ControllerHandle. Drivers will typically use the device path attached to
 688   ControllerHandle and/or the services from the bus I/O abstraction attached to
 689   ControllerHandle to determine if the driver supports ControllerHandle. This function
 690   may be called many times during platform initialization. In order to reduce boot times, the tests
 691   performed by this function must be very small, and take as little time as possible to execute. This
 692   function must not change the state of any hardware devices, and this function must be aware that the
 693   device specified by ControllerHandle may already be managed by the same driver or a
 694   different driver. This function must match its calls to AllocatePages() with FreePages(),
 695   AllocatePool() with FreePool(), and OpenProtocol() with CloseProtocol().
 696   Since ControllerHandle may have been previously started by the same driver, if a protocol is
 697   already in the opened state, then it must not be closed with CloseProtocol(). This is required
 698   to guarantee the state of ControllerHandle is not modified by this function.
 699
 700   @param[in]  This                 A pointer to the EFI_DRIVER_BINDING_PROTOCOL instance.
 701   @param[in]  ControllerHandle     The handle of the controller to test. This handle
 702                                    must support a protocol interface that supplies
 703                                    an I/O abstraction to the driver.
 704   @param[in]  RemainingDevicePath  A pointer to the remaining portion of a device path.  This
 705                                    parameter is ignored by device drivers, and is optional for bus
 706                                    drivers. For bus drivers, if this parameter is not NULL, then
 707                                    the bus driver must determine if the bus controller specified
 708                                    by ControllerHandle and the child controller specified
 709                                    by RemainingDevicePath are both supported by this
 710                                    bus driver.
 711
 712   @retval EFI_SUCCESS              The device specified by ControllerHandle and
 713                                    RemainingDevicePath is supported by the driver specified by This.
 714   @retval EFI_ALREADY_STARTED      The device specified by ControllerHandle and
 715                                    RemainingDevicePath is already being managed by the driver
 716                                    specified by This.
 717   @retval EFI_ACCESS_DENIED        The device specified by ControllerHandle and
 718                                    RemainingDevicePath is already being managed by a different
 719                                    driver or an application that requires exclusive access.
 720                                    Currently not implemented.
 721   @retval EFI_UNSUPPORTED          The device specified by ControllerHandle and
 722                                    RemainingDevicePath is not supported by the driver specified by This.
 723 **/
 724 EFI_STATUS
 725 EFIAPI
 726 NvmExpressDriverBindingSupported (
 727   IN EFI_DRIVER_BINDING_PROTOCOL  *This,
 728   IN EFI_HANDLE                   Controller,
 729   IN EFI_DEVICE_PATH_PROTOCOL     *RemainingDevicePath
 730   )
 731 {
 732   EFI_STATUS                Status;
 733   EFI_DEV_PATH_PTR          DevicePathNode;
 734   EFI_DEVICE_PATH_PROTOCOL  *ParentDevicePath;
 735   EFI_PCI_IO_PROTOCOL       *PciIo;
 736   UINT8                     ClassCode[3];
 737
 738   //
 739   // Check whether device path is valid
 740   //
 741   if (RemainingDevicePath != NULL) {
 742     //
 743     // Check if RemainingDevicePath is the End of Device Path Node,
 744     // if yes, go on checking other conditions
 745     //
 746     if (!IsDevicePathEnd (RemainingDevicePath)) {
 747       //
 748       // If RemainingDevicePath isn't the End of Device Path Node,
 749       // check its validation
 750       //
 751       DevicePathNode.DevPath = RemainingDevicePath;
 752
 753       if ((DevicePathNode.DevPath->Type    != MESSAGING_DEVICE_PATH) ||
 754           (DevicePathNode.DevPath->SubType != MSG_NVME_NAMESPACE_DP) ||
 755           (DevicePathNodeLength(DevicePathNode.DevPath) != sizeof(NVME_NAMESPACE_DEVICE_PATH))) {
 756          return EFI_UNSUPPORTED;
 757       }
 758     }
 759   }
 760
 761   //
 762   // Open the EFI Device Path protocol needed to perform the supported test
 763   //
 764   Status = gBS->OpenProtocol (
 765                   Controller,
 766                   &gEfiDevicePathProtocolGuid,
 767                   (VOID **) &ParentDevicePath,
 768                   This->DriverBindingHandle,
 769                   Controller,
 770                   EFI_OPEN_PROTOCOL_BY_DRIVER
 771                   );
 772   if (Status == EFI_ALREADY_STARTED) {
 773     return EFI_SUCCESS;
 774   }
 775
 776   if (EFI_ERROR (Status)) {
 777     return Status;
 778   }
 779
 780   //
 781   // Close protocol, don't use device path protocol in the Support() function
 782   //
 783   gBS->CloseProtocol (
 784          Controller,
 785          &gEfiDevicePathProtocolGuid,
 786          This->DriverBindingHandle,
 787          Controller
 788          );
 789
 790   //
 791   // Attempt to Open PCI I/O Protocol
 792   //
 793   Status = gBS->OpenProtocol (
 794                   Controller,
 795                   &gEfiPciIoProtocolGuid,
 796                   (VOID **) &PciIo,
 797                   This->DriverBindingHandle,
 798                   Controller,
 799                   EFI_OPEN_PROTOCOL_BY_DRIVER
 800                   );
 801   if (Status == EFI_ALREADY_STARTED) {
 802     return EFI_SUCCESS;
 803   }
 804
 805   if (EFI_ERROR (Status)) {
 806     return Status;
 807   }
 808
 809   //
 810   // Now further check the PCI header: Base class (offset 0x0B) and Sub Class (offset 0x0A).
 811   // This controller should be a Nvm Express controller.
 812   //
 813   Status = PciIo->Pci.Read (
 814                         PciIo,
 815                         EfiPciIoWidthUint8,
 816                         PCI_CLASSCODE_OFFSET,
 817                         sizeof (ClassCode),
 818                         ClassCode
 819                         );
 820   if (EFI_ERROR (Status)) {
 821     goto Done;
 822   }
 823
 824   //
 825   // Examine Nvm Express controller PCI Configuration table fields
 826   //
 827   if ((ClassCode[0] != PCI_IF_NVMHCI) || (ClassCode[1] != PCI_CLASS_MASS_STORAGE_NVM) || (ClassCode[2] != PCI_CLASS_MASS_STORAGE)) {
 828     Status = EFI_UNSUPPORTED;
 829   }
 830
 831 Done:
 832   gBS->CloseProtocol (
 833          Controller,
 834          &gEfiPciIoProtocolGuid,
 835          This->DriverBindingHandle,
 836          Controller
 837          );
 838
 839   return Status;
 840 }
 841
 842
 843 /**
 844   Starts a device controller or a bus controller.
 845
 846   The Start() function is designed to be invoked from the EFI boot service ConnectController().
 847   As a result, much of the error checking on the parameters to Start() has been moved into this
 848   common boot service. It is legal to call Start() from other locations,
 849   but the following calling restrictions must be followed or the system behavior will not be deterministic.
 850   1. ControllerHandle must be a valid EFI_HANDLE.
 851   2. If RemainingDevicePath is not NULL, then it must be a pointer to a naturally aligned
 852      EFI_DEVICE_PATH_PROTOCOL.
 853   3. Prior to calling Start(), the Supported() function for the driver specified by This must
 854      have been called with the same calling parameters, and Supported() must have returned EFI_SUCCESS.
 855
 856   @param[in]  This                 A pointer to the EFI_DRIVER_BINDING_PROTOCOL instance.
 857   @param[in]  ControllerHandle     The handle of the controller to start. This handle
 858                                    must support a protocol interface that supplies
 859                                    an I/O abstraction to the driver.
 860   @param[in]  RemainingDevicePath  A pointer to the remaining portion of a device path.  This
 861                                    parameter is ignored by device drivers, and is optional for bus
 862                                    drivers. For a bus driver, if this parameter is NULL, then handles
 863                                    for all the children of Controller are created by this driver.
 864                                    If this parameter is not NULL and the first Device Path Node is
 865                                    not the End of Device Path Node, then only the handle for the
 866                                    child device specified by the first Device Path Node of
 867                                    RemainingDevicePath is created by this driver.
 868                                    If the first Device Path Node of RemainingDevicePath is
 869                                    the End of Device Path Node, no child handle is created by this
 870                                    driver.
 871
 872   @retval EFI_SUCCESS              The device was started.
 873   @retval EFI_DEVICE_ERROR         The device could not be started due to a device error.Currently not implemented.
 874   @retval EFI_OUT_OF_RESOURCES     The request could not be completed due to a lack of resources.
 875   @retval Others                   The driver failded to start the device.
 876
 877 **/
 878 EFI_STATUS
 879 EFIAPI
 880 NvmExpressDriverBindingStart (
 881   IN EFI_DRIVER_BINDING_PROTOCOL  *This,
 882   IN EFI_HANDLE                   Controller,
 883   IN EFI_DEVICE_PATH_PROTOCOL     *RemainingDevicePath
 884   )
 885 {
 886   EFI_STATUS                          Status;
 887   EFI_PCI_IO_PROTOCOL                 *PciIo;
 888   NVME_CONTROLLER_PRIVATE_DATA        *Private;
 889   EFI_DEVICE_PATH_PROTOCOL            *ParentDevicePath;
 890   UINT32                              NamespaceId;
 891   EFI_PHYSICAL_ADDRESS                MappedAddr;
 892   UINTN                               Bytes;
 893   EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL  *Passthru;
 894
 895   DEBUG ((EFI_D_INFO, "NvmExpressDriverBindingStart: start\n"));
 896
 897   Private          = NULL;
 898   Passthru         = NULL;
 899   ParentDevicePath = NULL;
 900
 901   Status = gBS->OpenProtocol (
 902                   Controller,
 903                   &gEfiDevicePathProtocolGuid,
 904                   (VOID **) &ParentDevicePath,
 905                   This->DriverBindingHandle,
 906                   Controller,
 907                   EFI_OPEN_PROTOCOL_BY_DRIVER
 908                   );
 909   if ((EFI_ERROR (Status)) && (Status != EFI_ALREADY_STARTED)) {
 910     return Status;
 911   }
 912
 913   Status = gBS->OpenProtocol (
 914                   Controller,
 915                   &gEfiPciIoProtocolGuid,
 916                   (VOID **) &PciIo,
 917                   This->DriverBindingHandle,
 918                   Controller,
 919                   EFI_OPEN_PROTOCOL_BY_DRIVER
 920                   );
 921
 922   if (EFI_ERROR (Status) && (Status != EFI_ALREADY_STARTED)) {
 923     return Status;
 924   }
 925
 926   //
 927   // Check EFI_ALREADY_STARTED to reuse the original NVME_CONTROLLER_PRIVATE_DATA.
 928   //
 929   if (Status != EFI_ALREADY_STARTED) {
 930     Private = AllocateZeroPool (sizeof (NVME_CONTROLLER_PRIVATE_DATA));
 931
 932     if (Private == NULL) {
 933       DEBUG ((EFI_D_ERROR, "NvmExpressDriverBindingStart: allocating pool for Nvme Private Data failed!\n"));
 934       Status = EFI_OUT_OF_RESOURCES;
 935       goto Exit;
 936     }
 937
 938     //
 939     // 6 x 4kB aligned buffers will be carved out of this buffer.
 940     // 1st 4kB boundary is the start of the admin submission queue.
 941     // 2nd 4kB boundary is the start of the admin completion queue.
 942     // 3rd 4kB boundary is the start of I/O submission queue #1.
 943     // 4th 4kB boundary is the start of I/O completion queue #1.
 944     // 5th 4kB boundary is the start of I/O submission queue #2.
 945     // 6th 4kB boundary is the start of I/O completion queue #2.
 946     //
 947     // Allocate 6 pages of memory, then map it for bus master read and write.
 948     //
 949     Status = PciIo->AllocateBuffer (
 950                       PciIo,
 951                       AllocateAnyPages,
 952                       EfiBootServicesData,
 953                       6,
 954                       (VOID**)&Private->Buffer,
 955                       0
 956                       );
 957     if (EFI_ERROR (Status)) {
 958       goto Exit;
 959     }
 960
 961     Bytes = EFI_PAGES_TO_SIZE (6);
 962     Status = PciIo->Map (
 963                       PciIo,
 964                       EfiPciIoOperationBusMasterCommonBuffer,
 965                       Private->Buffer,
 966                       &Bytes,
 967                       &MappedAddr,
 968                       &Private->Mapping
 969                       );
 970
 971     if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (6))) {
 972       goto Exit;
 973     }
 974
 975     Private->BufferPciAddr = (UINT8 *)(UINTN)MappedAddr;
 976
 977     Private->Signature = NVME_CONTROLLER_PRIVATE_DATA_SIGNATURE;
 978     Private->ControllerHandle          = Controller;
 979     Private->ImageHandle               = This->DriverBindingHandle;
 980     Private->DriverBindingHandle       = This->DriverBindingHandle;
 981     Private->PciIo                     = PciIo;
 982     Private->ParentDevicePath          = ParentDevicePath;
 983     Private->Passthru.Mode             = &Private->PassThruMode;
 984     Private->Passthru.PassThru         = NvmExpressPassThru;
 985     Private->Passthru.GetNextNamespace = NvmExpressGetNextNamespace;
 986     Private->Passthru.BuildDevicePath  = NvmExpressBuildDevicePath;
 987     Private->Passthru.GetNamespace     = NvmExpressGetNamespace;
 988     CopyMem (&Private->PassThruMode, &gEfiNvmExpressPassThruMode, sizeof (EFI_NVM_EXPRESS_PASS_THRU_MODE));
 989     InitializeListHead (&Private->AsyncPassThruQueue);
 990     InitializeListHead (&Private->UnsubmittedSubtasks);
 991
 992     Status = NvmeControllerInit (Private);
 993     if (EFI_ERROR(Status)) {
 994       goto Exit;
 995     }
 996
 997     //
 998     // Start the asynchronous I/O completion monitor
 999     //
1000     Status = gBS->CreateEvent (
1001                     EVT_TIMER | EVT_NOTIFY_SIGNAL,
1002                     TPL_NOTIFY,
1003                     ProcessAsyncTaskList,
1004                     Private,
1005                     &Private->TimerEvent
1006                     );
1007     if (EFI_ERROR (Status)) {
1008       goto Exit;
1009     }
1010
1011     Status = gBS->SetTimer (
1012                     Private->TimerEvent,
1013                     TimerPeriodic,
1014                     NVME_HC_ASYNC_TIMER
1015                     );
1016     if (EFI_ERROR (Status)) {
1017       goto Exit;
1018     }
1019
1020     Status = gBS->InstallMultipleProtocolInterfaces (
1021                     &Controller,
1022                     &gEfiNvmExpressPassThruProtocolGuid,
1023                     &Private->Passthru,
1024                     NULL
1025                     );
1026     if (EFI_ERROR (Status)) {
1027       goto Exit;
1028     }
1029   } else {
1030     Status = gBS->OpenProtocol (
1031                     Controller,
1032                     &gEfiNvmExpressPassThruProtocolGuid,
1033                     (VOID **) &Passthru,
1034                     This->DriverBindingHandle,
1035                     Controller,
1036                     EFI_OPEN_PROTOCOL_GET_PROTOCOL
1037                     );
1038     if (EFI_ERROR (Status)) {
1039       goto Exit;
1040     }
1041
1042     Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (Passthru);
1043   }
1044
1045   if (RemainingDevicePath == NULL) {
1046     //
1047     // Enumerate all NVME namespaces in the controller
1048     //
1049     Status = DiscoverAllNamespaces (
1050                Private
1051                );
1052
1053   } else if (!IsDevicePathEnd (RemainingDevicePath)) {
1054     //
1055     // Enumerate the specified NVME namespace
1056     //
1057     Status = Private->Passthru.GetNamespace (
1058                                  &Private->Passthru,
1059                                  RemainingDevicePath,
1060                                  &NamespaceId
1061                                  );
1062
1063     if (!EFI_ERROR (Status)) {
1064       Status = EnumerateNvmeDevNamespace (
1065                  Private,
1066                  NamespaceId
1067                  );
1068     }
1069   }
1070
1071   DEBUG ((EFI_D_INFO, "NvmExpressDriverBindingStart: end successfully\n"));
1072   return EFI_SUCCESS;
1073
1074 Exit:
1075   if ((Private != NULL) && (Private->Mapping != NULL)) {
1076     PciIo->Unmap (PciIo, Private->Mapping);
1077   }
1078
1079   if ((Private != NULL) && (Private->Buffer != NULL)) {
1080     PciIo->FreeBuffer (PciIo, 6, Private->Buffer);
1081   }
1082
1083   if ((Private != NULL) && (Private->ControllerData != NULL)) {
1084     FreePool (Private->ControllerData);
1085   }
1086
1087   if (Private != NULL) {
1088     if (Private->TimerEvent != NULL) {
1089       gBS->CloseEvent (Private->TimerEvent);
1090     }
1091
1092     FreePool (Private);
1093   }
1094
1095   gBS->CloseProtocol (
1096          Controller,
1097          &gEfiPciIoProtocolGuid,
1098          This->DriverBindingHandle,
1099          Controller
1100          );
1101
1102   gBS->CloseProtocol (
1103          Controller,
1104          &gEfiDevicePathProtocolGuid,
1105          This->DriverBindingHandle,
1106          Controller
1107          );
1108
1109   DEBUG ((EFI_D_INFO, "NvmExpressDriverBindingStart: end with %r\n", Status));
1110
1111   return Status;
1112 }
1113
1114
1115 /**
1116   Stops a device controller or a bus controller.
1117
1118   The Stop() function is designed to be invoked from the EFI boot service DisconnectController().
1119   As a result, much of the error checking on the parameters to Stop() has been moved
1120   into this common boot service. It is legal to call Stop() from other locations,
1121   but the following calling restrictions must be followed or the system behavior will not be deterministic.
1122   1. ControllerHandle must be a valid EFI_HANDLE that was used on a previous call to this
1123      same driver's Start() function.
1124   2. The first NumberOfChildren handles of ChildHandleBuffer must all be a valid
1125      EFI_HANDLE. In addition, all of these handles must have been created in this driver's
1126      Start() function, and the Start() function must have called OpenProtocol() on
1127      ControllerHandle with an Attribute of EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER.
1128
1129   @param[in]  This              A pointer to the EFI_DRIVER_BINDING_PROTOCOL instance.
1130   @param[in]  ControllerHandle  A handle to the device being stopped. The handle must
1131                                 support a bus specific I/O protocol for the driver
1132                                 to use to stop the device.
1133   @param[in]  NumberOfChildren  The number of child device handles in ChildHandleBuffer.
1134   @param[in]  ChildHandleBuffer An array of child handles to be freed. May be NULL
1135                                 if NumberOfChildren is 0.
1136
1137   @retval EFI_SUCCESS           The device was stopped.
1138   @retval EFI_DEVICE_ERROR      The device could not be stopped due to a device error.
1139
1140 **/
1141 EFI_STATUS
1142 EFIAPI
1143 NvmExpressDriverBindingStop (
1144   IN  EFI_DRIVER_BINDING_PROTOCOL     *This,
1145   IN  EFI_HANDLE                      Controller,
1146   IN  UINTN                           NumberOfChildren,
1147   IN  EFI_HANDLE                      *ChildHandleBuffer
1148   )
1149 {
1150   EFI_STATUS                          Status;
1151   BOOLEAN                             AllChildrenStopped;
1152   UINTN                               Index;
1153   NVME_CONTROLLER_PRIVATE_DATA        *Private;
1154   EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL  *PassThru;
1155   BOOLEAN                             IsEmpty;
1156   EFI_TPL                             OldTpl;
1157
1158   if (NumberOfChildren == 0) {
1159     Status = gBS->OpenProtocol (
1160                     Controller,
1161                     &gEfiNvmExpressPassThruProtocolGuid,
1162                     (VOID **) &PassThru,
1163                     This->DriverBindingHandle,
1164                     Controller,
1165                     EFI_OPEN_PROTOCOL_GET_PROTOCOL
1166                     );
1167
1168     if (!EFI_ERROR (Status)) {
1169       Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (PassThru);
1170
1171       //
1172       // Wait for the asynchronous PassThru queue to become empty.
1173       //
1174       while (TRUE) {
1175         OldTpl  = gBS->RaiseTPL (TPL_NOTIFY);
1176         IsEmpty = IsListEmpty (&Private->AsyncPassThruQueue) &&
1177                   IsListEmpty (&Private->UnsubmittedSubtasks);
1178         gBS->RestoreTPL (OldTpl);
1179
1180         if (IsEmpty) {
1181           break;
1182         }
1183
1184         gBS->Stall (100);
1185       }
1186
1187       gBS->UninstallMultipleProtocolInterfaces (
1188             Controller,
1189             &gEfiNvmExpressPassThruProtocolGuid,
1190             PassThru,
1191             NULL
1192             );
1193
1194       if (Private->TimerEvent != NULL) {
1195         gBS->CloseEvent (Private->TimerEvent);
1196       }
1197
1198       if (Private->Mapping != NULL) {
1199         Private->PciIo->Unmap (Private->PciIo, Private->Mapping);
1200       }
1201
1202       if (Private->Buffer != NULL) {
1203         Private->PciIo->FreeBuffer (Private->PciIo, 6, Private->Buffer);
1204       }
1205
1206       FreePool (Private->ControllerData);
1207       FreePool (Private);
1208     }
1209
1210     gBS->CloseProtocol (
1211           Controller,
1212           &gEfiPciIoProtocolGuid,
1213           This->DriverBindingHandle,
1214           Controller
1215           );
1216     gBS->CloseProtocol (
1217           Controller,
1218           &gEfiDevicePathProtocolGuid,
1219           This->DriverBindingHandle,
1220           Controller
1221           );
1222     return EFI_SUCCESS;
1223   }
1224
1225   AllChildrenStopped = TRUE;
1226
1227   for (Index = 0; Index < NumberOfChildren; Index++) {
1228     Status = UnregisterNvmeNamespace (This, Controller, ChildHandleBuffer[Index]);
1229     if (EFI_ERROR (Status)) {
1230       AllChildrenStopped = FALSE;
1231     }
1232   }
1233
1234   if (!AllChildrenStopped) {
1235     return EFI_DEVICE_ERROR;
1236   }
1237
1238   return EFI_SUCCESS;
1239 }
1240
1241 /**
1242   This is the unload handle for the NVM Express driver.
1243
1244   Disconnect the driver specified by ImageHandle from the NVMe device in the handle database.
1245   Uninstall all the protocols installed in the driver.
1246
1247   @param[in]  ImageHandle       The drivers' driver image.
1248
1249   @retval EFI_SUCCESS           The image is unloaded.
1250   @retval Others                Failed to unload the image.
1251
1252 **/
1253 EFI_STATUS
1254 EFIAPI
1255 NvmExpressUnload (
1256   IN EFI_HANDLE             ImageHandle
1257   )
1258 {
1259   EFI_STATUS                        Status;
1260   EFI_HANDLE                        *DeviceHandleBuffer;
1261   UINTN                             DeviceHandleCount;
1262   UINTN                             Index;
1263   EFI_COMPONENT_NAME_PROTOCOL       *ComponentName;
1264   EFI_COMPONENT_NAME2_PROTOCOL      *ComponentName2;
1265
1266   //
1267   // Get the list of the device handles managed by this driver.
1268   // If there is an error getting the list, then means the driver
1269   // doesn't manage any device. At this way, we would only close
1270   // those protocols installed at image handle.
1271   //
1272   DeviceHandleBuffer = NULL;
1273   Status = gBS->LocateHandleBuffer (
1274                   ByProtocol,
1275                   &gEfiNvmExpressPassThruProtocolGuid,
1276                   NULL,
1277                   &DeviceHandleCount,
1278                   &DeviceHandleBuffer
1279                   );
1280
1281   if (!EFI_ERROR (Status)) {
1282     //
1283     // Disconnect the driver specified by ImageHandle from all
1284     // the devices in the handle database.
1285     //
1286     for (Index = 0; Index < DeviceHandleCount; Index++) {
1287       Status = gBS->DisconnectController (
1288                       DeviceHandleBuffer[Index],
1289                       ImageHandle,
1290                       NULL
1291                       );
1292       if (EFI_ERROR (Status)) {
1293         goto EXIT;
1294       }
1295     }
1296   }
1297
1298   //
1299   // Uninstall all the protocols installed in the driver entry point
1300   //
1301   Status = gBS->UninstallMultipleProtocolInterfaces (
1302                   ImageHandle,
1303                   &gEfiDriverBindingProtocolGuid,
1304                   &gNvmExpressDriverBinding,
1305                   &gEfiDriverSupportedEfiVersionProtocolGuid,
1306                   &gNvmExpressDriverSupportedEfiVersion,
1307                   NULL
1308                   );
1309
1310   if (EFI_ERROR (Status)) {
1311     goto EXIT;
1312   }
1313
1314   //
1315   // Note we have to one by one uninstall the following protocols.
1316   // It's because some of them are optionally installed based on
1317   // the following PCD settings.
1318   //   gEfiMdePkgTokenSpaceGuid.PcdDriverDiagnosticsDisable
1319   //   gEfiMdePkgTokenSpaceGuid.PcdComponentNameDisable
1320   //   gEfiMdePkgTokenSpaceGuid.PcdDriverDiagnostics2Disable
1321   //   gEfiMdePkgTokenSpaceGuid.PcdComponentName2Disable
1322   //
1323   Status = gBS->HandleProtocol (
1324                   ImageHandle,
1325                   &gEfiComponentNameProtocolGuid,
1326                   (VOID **) &ComponentName
1327                   );
1328   if (!EFI_ERROR (Status)) {
1329     gBS->UninstallProtocolInterface (
1330            ImageHandle,
1331            &gEfiComponentNameProtocolGuid,
1332            ComponentName
1333            );
1334   }
1335
1336   Status = gBS->HandleProtocol (
1337                   ImageHandle,
1338                   &gEfiComponentName2ProtocolGuid,
1339                   (VOID **) &ComponentName2
1340                   );
1341   if (!EFI_ERROR (Status)) {
1342     gBS->UninstallProtocolInterface (
1343            ImageHandle,
1344            &gEfiComponentName2ProtocolGuid,
1345            ComponentName2
1346            );
1347   }
1348
1349   Status = EFI_SUCCESS;
1350
1351 EXIT:
1352   //
1353   // Free the buffer containing the list of handles from the handle database
1354   //
1355   if (DeviceHandleBuffer != NULL) {
1356     gBS->FreePool (DeviceHandleBuffer);
1357   }
1358   return Status;
1359 }
1360
1361 /**
1362   The entry point for Nvm Express driver, used to install Nvm Express driver on the ImageHandle.
1363
1364   @param  ImageHandle   The firmware allocated handle for this driver image.
1365   @param  SystemTable   Pointer to the EFI system table.
1366
1367   @retval EFI_SUCCESS   Driver loaded.
1368   @retval other         Driver not loaded.
1369
1370 **/
1371 EFI_STATUS
1372 EFIAPI
1373 NvmExpressDriverEntry (
1374   IN EFI_HANDLE        ImageHandle,
1375   IN EFI_SYSTEM_TABLE  *SystemTable
1376   )
1377 {
1378   EFI_STATUS              Status;
1379
1380   Status = EfiLibInstallDriverBindingComponentName2 (
1381              ImageHandle,
1382              SystemTable,
1383              &gNvmExpressDriverBinding,
1384              ImageHandle,
1385              &gNvmExpressComponentName,
1386              &gNvmExpressComponentName2
1387              );
1388   ASSERT_EFI_ERROR (Status);
1389
1390   //
1391   // Install EFI Driver Supported EFI Version Protocol required for
1392   // EFI drivers that are on PCI and other plug in cards.
1393   //
1394   gNvmExpressDriverSupportedEfiVersion.FirmwareVersion = 0x00020028;
1395   Status = gBS->InstallMultipleProtocolInterfaces (
1396                   &ImageHandle,
1397                   &gEfiDriverSupportedEfiVersionProtocolGuid,
1398                   &gNvmExpressDriverSupportedEfiVersion,
1399                   NULL
1400                   );
1401   ASSERT_EFI_ERROR (Status);
1402   return Status;
1403 }