MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.c

   1 /** @file
   2   NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows
   3   NVM Express specification.
   4
   5   Copyright (c) 2013 - 2016, Intel Corporation. All rights reserved.<BR>
   6   This program and the accompanying materials
   7   are licensed and made available under the terms and conditions of the BSD License
   8   which accompanies this distribution.  The full text of the license may be found at
   9   http://opensource.org/licenses/bsd-license.php.
  10
  11   THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  12   WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  13
  14 **/
  15
  16 #include "NvmExpress.h"
  17
  18 //
  19 // NVM Express Driver Binding Protocol Instance
  20 //
  21 EFI_DRIVER_BINDING_PROTOCOL gNvmExpressDriverBinding = {
  22   NvmExpressDriverBindingSupported,
  23   NvmExpressDriverBindingStart,
  24   NvmExpressDriverBindingStop,
  25   0x10,
  26   NULL,
  27   NULL
  28 };
  29
  30 //
  31 // NVM Express EFI Driver Supported EFI Version Protocol Instance
  32 //
  33 EFI_DRIVER_SUPPORTED_EFI_VERSION_PROTOCOL gNvmExpressDriverSupportedEfiVersion = {
  34   sizeof (EFI_DRIVER_SUPPORTED_EFI_VERSION_PROTOCOL), // Size of Protocol structure.
  35   0                                                   // Version number to be filled at start up.
  36 };
  37
  38 //
  39 // Template for NVM Express Pass Thru Mode data structure.
  40 //
  41 GLOBAL_REMOVE_IF_UNREFERENCED EFI_NVM_EXPRESS_PASS_THRU_MODE gEfiNvmExpressPassThruMode = {
  42   EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL   |
  43   EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL    |
  44   EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_NONBLOCKIO |
  45   EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_CMD_SET_NVM,
  46   sizeof (UINTN),
  47   0x10100
  48 };
  49
  50 /**
  51   Check if the specified Nvm Express device namespace is active, and create child handles
  52   for them with BlockIo and DiskInfo protocol instances.
  53
  54   @param[in] Private         The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
  55   @param[in] NamespaceId     The NVM Express namespace ID  for which a device path node is to be
  56                              allocated and built. Caller must set the NamespaceId to zero if the
  57                              device path node will contain a valid UUID.
  58
  59   @retval EFI_SUCCESS        All the namespaces in the device are successfully enumerated.
  60   @return Others             Some error occurs when enumerating the namespaces.
  61
  62 **/
  63 EFI_STATUS
  64 EnumerateNvmeDevNamespace (
  65   IN NVME_CONTROLLER_PRIVATE_DATA       *Private,
  66   UINT32                                NamespaceId
  67   )
  68 {
  69   NVME_ADMIN_NAMESPACE_DATA             *NamespaceData;
  70   EFI_DEVICE_PATH_PROTOCOL              *NewDevicePathNode;
  71   EFI_DEVICE_PATH_PROTOCOL              *DevicePath;
  72   EFI_HANDLE                            DeviceHandle;
  73   EFI_DEVICE_PATH_PROTOCOL              *ParentDevicePath;
  74   EFI_DEVICE_PATH_PROTOCOL              *RemainingDevicePath;
  75   NVME_DEVICE_PRIVATE_DATA              *Device;
  76   EFI_STATUS                            Status;
  77   UINT32                                Lbads;
  78   UINT32                                Flbas;
  79   UINT32                                LbaFmtIdx;
  80   UINT8                                 Sn[21];
  81   UINT8                                 Mn[41];
  82   VOID                                  *DummyInterface;
  83
  84   NewDevicePathNode = NULL;
  85   DevicePath        = NULL;
  86   Device            = NULL;
  87
  88   //
  89   // Allocate a buffer for Identify Namespace data
  90   //
  91   NamespaceData = AllocateZeroPool(sizeof (NVME_ADMIN_NAMESPACE_DATA));
  92   if(NamespaceData == NULL) {
  93     return EFI_OUT_OF_RESOURCES;
  94   }
  95
  96   ParentDevicePath = Private->ParentDevicePath;
  97   //
  98   // Identify Namespace
  99   //
 100   Status = NvmeIdentifyNamespace (
 101              Private,
 102              NamespaceId,
 103              (VOID *)NamespaceData
 104              );
 105   if (EFI_ERROR(Status)) {
 106     goto Exit;
 107   }
 108   //
 109   // Validate Namespace
 110   //
 111   if (NamespaceData->Ncap == 0) {
 112     Status = EFI_DEVICE_ERROR;
 113   } else {
 114     //
 115     // allocate device private data for each discovered namespace
 116     //
 117     Device = AllocateZeroPool(sizeof(NVME_DEVICE_PRIVATE_DATA));
 118     if (Device == NULL) {
 119       Status = EFI_OUT_OF_RESOURCES;
 120       goto Exit;
 121     }
 122
 123     //
 124     // Initialize SSD namespace instance data
 125     //
 126     Device->Signature           = NVME_DEVICE_PRIVATE_DATA_SIGNATURE;
 127     Device->NamespaceId         = NamespaceId;
 128     Device->NamespaceUuid       = NamespaceData->Eui64;
 129
 130     Device->ControllerHandle    = Private->ControllerHandle;
 131     Device->DriverBindingHandle = Private->DriverBindingHandle;
 132     Device->Controller          = Private;
 133
 134     //
 135     // Build BlockIo media structure
 136     //
 137     Device->Media.MediaId        = 0;
 138     Device->Media.RemovableMedia = FALSE;
 139     Device->Media.MediaPresent   = TRUE;
 140     Device->Media.LogicalPartition = FALSE;
 141     Device->Media.ReadOnly       = FALSE;
 142     Device->Media.WriteCaching   = FALSE;
 143     Device->Media.IoAlign        = Private->PassThruMode.IoAlign;
 144
 145     Flbas     = NamespaceData->Flbas;
 146     LbaFmtIdx = Flbas & 0xF;
 147     Lbads     = NamespaceData->LbaFormat[LbaFmtIdx].Lbads;
 148     Device->Media.BlockSize = (UINT32)1 << Lbads;
 149
 150     Device->Media.LastBlock                     = NamespaceData->Nsze - 1;
 151     Device->Media.LogicalBlocksPerPhysicalBlock = 1;
 152     Device->Media.LowestAlignedLba              = 1;
 153
 154     //
 155     // Create BlockIo Protocol instance
 156     //
 157     Device->BlockIo.Revision     = EFI_BLOCK_IO_PROTOCOL_REVISION2;
 158     Device->BlockIo.Media        = &Device->Media;
 159     Device->BlockIo.Reset        = NvmeBlockIoReset;
 160     Device->BlockIo.ReadBlocks   = NvmeBlockIoReadBlocks;
 161     Device->BlockIo.WriteBlocks  = NvmeBlockIoWriteBlocks;
 162     Device->BlockIo.FlushBlocks  = NvmeBlockIoFlushBlocks;
 163
 164     //
 165     // Create BlockIo2 Protocol instance
 166     //
 167     Device->BlockIo2.Media          = &Device->Media;
 168     Device->BlockIo2.Reset          = NvmeBlockIoResetEx;
 169     Device->BlockIo2.ReadBlocksEx   = NvmeBlockIoReadBlocksEx;
 170     Device->BlockIo2.WriteBlocksEx  = NvmeBlockIoWriteBlocksEx;
 171     Device->BlockIo2.FlushBlocksEx  = NvmeBlockIoFlushBlocksEx;
 172     InitializeListHead (&Device->AsyncQueue);
 173
 174     //
 175     // Create StorageSecurityProtocol Instance
 176     //
 177     Device->StorageSecurity.ReceiveData = NvmeStorageSecurityReceiveData;
 178     Device->StorageSecurity.SendData    = NvmeStorageSecuritySendData;
 179
 180     //
 181     // Create DiskInfo Protocol instance
 182     //
 183     CopyMem (&Device->NamespaceData, NamespaceData, sizeof (NVME_ADMIN_NAMESPACE_DATA));
 184     InitializeDiskInfo (Device);
 185
 186     //
 187     // Create a Nvm Express Namespace Device Path Node
 188     //
 189     Status = Private->Passthru.BuildDevicePath (
 190                                  &Private->Passthru,
 191                                  Device->NamespaceId,
 192                                  &NewDevicePathNode
 193                                  );
 194
 195     if (EFI_ERROR(Status)) {
 196       goto Exit;
 197     }
 198
 199     //
 200     // Append the SSD node to the controller's device path
 201     //
 202     DevicePath = AppendDevicePathNode (ParentDevicePath, NewDevicePathNode);
 203     if (DevicePath == NULL) {
 204       Status = EFI_OUT_OF_RESOURCES;
 205       goto Exit;
 206     }
 207
 208     DeviceHandle = NULL;
 209     RemainingDevicePath = DevicePath;
 210     Status = gBS->LocateDevicePath (&gEfiDevicePathProtocolGuid, &RemainingDevicePath, &DeviceHandle);
 211     if (!EFI_ERROR (Status) && (DeviceHandle != NULL) && IsDevicePathEnd(RemainingDevicePath)) {
 212       Status = EFI_ALREADY_STARTED;
 213       FreePool (DevicePath);
 214       goto Exit;
 215     }
 216
 217     Device->DevicePath = DevicePath;
 218
 219     //
 220     // Make sure the handle is NULL so we create a new handle
 221     //
 222     Device->DeviceHandle = NULL;
 223
 224     Status = gBS->InstallMultipleProtocolInterfaces (
 225                     &Device->DeviceHandle,
 226                     &gEfiDevicePathProtocolGuid,
 227                     Device->DevicePath,
 228                     &gEfiBlockIoProtocolGuid,
 229                     &Device->BlockIo,
 230                     &gEfiBlockIo2ProtocolGuid,
 231                     &Device->BlockIo2,
 232                     &gEfiDiskInfoProtocolGuid,
 233                     &Device->DiskInfo,
 234                     NULL
 235                     );
 236
 237     if(EFI_ERROR(Status)) {
 238       goto Exit;
 239     }
 240
 241     //
 242     // Check if the NVMe controller supports the Security Send and Security Receive commands
 243     //
 244     if ((Private->ControllerData->Oacs & SECURITY_SEND_RECEIVE_SUPPORTED) != 0) {
 245       Status = gBS->InstallProtocolInterface (
 246                       &Device->DeviceHandle,
 247                       &gEfiStorageSecurityCommandProtocolGuid,
 248                       EFI_NATIVE_INTERFACE,
 249                       &Device->StorageSecurity
 250                       );
 251       if(EFI_ERROR(Status)) {
 252         gBS->UninstallMultipleProtocolInterfaces (
 253                &Device->DeviceHandle,
 254                &gEfiDevicePathProtocolGuid,
 255                Device->DevicePath,
 256                &gEfiBlockIoProtocolGuid,
 257                &Device->BlockIo,
 258                &gEfiBlockIo2ProtocolGuid,
 259                &Device->BlockIo2,
 260                &gEfiDiskInfoProtocolGuid,
 261                &Device->DiskInfo,
 262                NULL
 263                );
 264         goto Exit;
 265       }
 266     }
 267
 268     gBS->OpenProtocol (
 269            Private->ControllerHandle,
 270            &gEfiNvmExpressPassThruProtocolGuid,
 271            (VOID **) &DummyInterface,
 272            Private->DriverBindingHandle,
 273            Device->DeviceHandle,
 274            EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER
 275            );
 276
 277     //
 278     // Dump NvmExpress Identify Namespace Data
 279     //
 280     DEBUG ((EFI_D_INFO, " == NVME IDENTIFY NAMESPACE [%d] DATA ==\n", NamespaceId));
 281     DEBUG ((EFI_D_INFO, "    NSZE        : 0x%x\n", NamespaceData->Nsze));
 282     DEBUG ((EFI_D_INFO, "    NCAP        : 0x%x\n", NamespaceData->Ncap));
 283     DEBUG ((EFI_D_INFO, "    NUSE        : 0x%x\n", NamespaceData->Nuse));
 284     DEBUG ((EFI_D_INFO, "    LBAF0.LBADS : 0x%x\n", (NamespaceData->LbaFormat[0].Lbads)));
 285
 286     //
 287     // Build controller name for Component Name (2) protocol.
 288     //
 289     CopyMem (Sn, Private->ControllerData->Sn, sizeof (Private->ControllerData->Sn));
 290     Sn[20] = 0;
 291     CopyMem (Mn, Private->ControllerData->Mn, sizeof (Private->ControllerData->Mn));
 292     Mn[40] = 0;
 293     UnicodeSPrintAsciiFormat (Device->ModelName, sizeof (Device->ModelName), "%a-%a-%x", Sn, Mn, NamespaceData->Eui64);
 294
 295     AddUnicodeString2 (
 296       "eng",
 297       gNvmExpressComponentName.SupportedLanguages,
 298       &Device->ControllerNameTable,
 299       Device->ModelName,
 300       TRUE
 301       );
 302
 303     AddUnicodeString2 (
 304       "en",
 305       gNvmExpressComponentName2.SupportedLanguages,
 306       &Device->ControllerNameTable,
 307       Device->ModelName,
 308       FALSE
 309       );
 310   }
 311
 312 Exit:
 313   if(NamespaceData != NULL) {
 314     FreePool (NamespaceData);
 315   }
 316
 317   if (NewDevicePathNode != NULL) {
 318     FreePool (NewDevicePathNode);
 319   }
 320
 321   if(EFI_ERROR(Status) && (Device != NULL) && (Device->DevicePath != NULL)) {
 322     FreePool (Device->DevicePath);
 323   }
 324   if(EFI_ERROR(Status) && (Device != NULL)) {
 325     FreePool (Device);
 326   }
 327   return Status;
 328 }
 329
 330 /**
 331   Discover all Nvm Express device namespaces, and create child handles for them with BlockIo
 332   and DiskInfo protocol instances.
 333
 334   @param[in] Private         The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 335
 336   @retval EFI_SUCCESS        All the namespaces in the device are successfully enumerated.
 337   @return Others             Some error occurs when enumerating the namespaces.
 338
 339 **/
 340 EFI_STATUS
 341 DiscoverAllNamespaces (
 342   IN NVME_CONTROLLER_PRIVATE_DATA       *Private
 343   )
 344 {
 345   EFI_STATUS                            Status;
 346   UINT32                                NamespaceId;
 347   EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL    *Passthru;
 348
 349   NamespaceId   = 0xFFFFFFFF;
 350   Passthru      = &Private->Passthru;
 351
 352   while (TRUE) {
 353     Status = Passthru->GetNextNamespace (
 354                          Passthru,
 355                          (UINT32 *)&NamespaceId
 356                          );
 357
 358     if (EFI_ERROR (Status)) {
 359       break;
 360     }
 361
 362     Status = EnumerateNvmeDevNamespace (
 363                Private,
 364                NamespaceId
 365                );
 366
 367     if (EFI_ERROR(Status)) {
 368       continue;
 369     }
 370   }
 371
 372   return EFI_SUCCESS;
 373 }
 374
 375 /**
 376   Unregisters a Nvm Express device namespace.
 377
 378   This function removes the protocols installed on the controller handle and
 379   frees the resources allocated for the namespace.
 380
 381   @param  This                  The pointer to EFI_DRIVER_BINDING_PROTOCOL instance.
 382   @param  Controller            The controller handle of the namespace.
 383   @param  Handle                The child handle.
 384
 385   @retval EFI_SUCCESS           The namespace is successfully unregistered.
 386   @return Others                Some error occurs when unregistering the namespace.
 387
 388 **/
 389 EFI_STATUS
 390 UnregisterNvmeNamespace (
 391   IN  EFI_DRIVER_BINDING_PROTOCOL    *This,
 392   IN  EFI_HANDLE                     Controller,
 393   IN  EFI_HANDLE                     Handle
 394   )
 395 {
 396   EFI_STATUS                               Status;
 397   EFI_BLOCK_IO_PROTOCOL                    *BlockIo;
 398   NVME_DEVICE_PRIVATE_DATA                 *Device;
 399   EFI_STORAGE_SECURITY_COMMAND_PROTOCOL    *StorageSecurity;
 400   BOOLEAN                                  IsEmpty;
 401   EFI_TPL                                  OldTpl;
 402   VOID                                     *DummyInterface;
 403
 404   BlockIo = NULL;
 405
 406   Status = gBS->OpenProtocol (
 407                   Handle,
 408                   &gEfiBlockIoProtocolGuid,
 409                   (VOID **) &BlockIo,
 410                   This->DriverBindingHandle,
 411                   Controller,
 412                   EFI_OPEN_PROTOCOL_GET_PROTOCOL
 413                   );
 414   if (EFI_ERROR (Status)) {
 415     return Status;
 416   }
 417
 418   Device  = NVME_DEVICE_PRIVATE_DATA_FROM_BLOCK_IO (BlockIo);
 419
 420   //
 421   // Wait for the device's asynchronous I/O queue to become empty.
 422   //
 423   while (TRUE) {
 424     OldTpl  = gBS->RaiseTPL (TPL_NOTIFY);
 425     IsEmpty = IsListEmpty (&Device->AsyncQueue);
 426     gBS->RestoreTPL (OldTpl);
 427
 428     if (IsEmpty) {
 429       break;
 430     }
 431
 432     gBS->Stall (100);
 433   }
 434
 435   //
 436   // Close the child handle
 437   //
 438   gBS->CloseProtocol (
 439          Controller,
 440          &gEfiNvmExpressPassThruProtocolGuid,
 441          This->DriverBindingHandle,
 442          Handle
 443          );
 444
 445   //
 446   // The Nvm Express driver installs the BlockIo and DiskInfo in the DriverBindingStart().
 447   // Here should uninstall both of them.
 448   //
 449   Status = gBS->UninstallMultipleProtocolInterfaces (
 450                   Handle,
 451                   &gEfiDevicePathProtocolGuid,
 452                   Device->DevicePath,
 453                   &gEfiBlockIoProtocolGuid,
 454                   &Device->BlockIo,
 455                   &gEfiBlockIo2ProtocolGuid,
 456                   &Device->BlockIo2,
 457                   &gEfiDiskInfoProtocolGuid,
 458                   &Device->DiskInfo,
 459                   NULL
 460                   );
 461
 462   if (EFI_ERROR (Status)) {
 463     gBS->OpenProtocol (
 464            Controller,
 465            &gEfiNvmExpressPassThruProtocolGuid,
 466            (VOID **) &DummyInterface,
 467            This->DriverBindingHandle,
 468            Handle,
 469            EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER
 470            );
 471     return Status;
 472   }
 473
 474   //
 475   // If Storage Security Command Protocol is installed, then uninstall this protocol.
 476   //
 477   Status = gBS->OpenProtocol (
 478                   Handle,
 479                   &gEfiStorageSecurityCommandProtocolGuid,
 480                   (VOID **) &StorageSecurity,
 481                   This->DriverBindingHandle,
 482                   Controller,
 483                   EFI_OPEN_PROTOCOL_GET_PROTOCOL
 484                   );
 485
 486   if (!EFI_ERROR (Status)) {
 487     Status = gBS->UninstallProtocolInterface (
 488                     Handle,
 489                     &gEfiStorageSecurityCommandProtocolGuid,
 490                     &Device->StorageSecurity
 491                     );
 492     if (EFI_ERROR (Status)) {
 493       gBS->OpenProtocol (
 494         Controller,
 495         &gEfiNvmExpressPassThruProtocolGuid,
 496         (VOID **) &DummyInterface,
 497         This->DriverBindingHandle,
 498         Handle,
 499         EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER
 500         );
 501       return Status;
 502     }
 503   }
 504
 505   if(Device->DevicePath != NULL) {
 506     FreePool (Device->DevicePath);
 507   }
 508
 509   if (Device->ControllerNameTable != NULL) {
 510     FreeUnicodeStringTable (Device->ControllerNameTable);
 511   }
 512
 513   FreePool (Device);
 514
 515   return EFI_SUCCESS;
 516 }
 517
 518 /**
 519   Call back function when the timer event is signaled.
 520
 521   @param[in]  Event     The Event this notify function registered to.
 522   @param[in]  Context   Pointer to the context data registered to the
 523                         Event.
 524
 525 **/
 526 VOID
 527 EFIAPI
 528 ProcessAsyncTaskList (
 529   IN EFI_EVENT                    Event,
 530   IN VOID*                        Context
 531   )
 532 {
 533   NVME_CONTROLLER_PRIVATE_DATA         *Private;
 534   EFI_PCI_IO_PROTOCOL                  *PciIo;
 535   NVME_CQ                              *Cq;
 536   UINT16                               QueueId;
 537   UINT32                               Data;
 538   LIST_ENTRY                           *Link;
 539   LIST_ENTRY                           *NextLink;
 540   NVME_PASS_THRU_ASYNC_REQ             *AsyncRequest;
 541   NVME_BLKIO2_SUBTASK                  *Subtask;
 542   NVME_BLKIO2_REQUEST                  *BlkIo2Request;
 543   EFI_BLOCK_IO2_TOKEN                  *Token;
 544   BOOLEAN                              HasNewItem;
 545   EFI_STATUS                           Status;
 546
 547   Private    = (NVME_CONTROLLER_PRIVATE_DATA*)Context;
 548   QueueId    = 2;
 549   Cq         = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
 550   HasNewItem = FALSE;
 551   PciIo      = Private->PciIo;
 552
 553   //
 554   // Submit asynchronous subtasks to the NVMe Submission Queue
 555   //
 556   for (Link = GetFirstNode (&Private->UnsubmittedSubtasks);
 557        !IsNull (&Private->UnsubmittedSubtasks, Link);
 558        Link = NextLink) {
 559     NextLink      = GetNextNode (&Private->UnsubmittedSubtasks, Link);
 560     Subtask       = NVME_BLKIO2_SUBTASK_FROM_LINK (Link);
 561     BlkIo2Request = Subtask->BlockIo2Request;
 562     Token         = BlkIo2Request->Token;
 563     RemoveEntryList (Link);
 564     BlkIo2Request->UnsubmittedSubtaskNum--;
 565
 566     //
 567     // If any previous subtask fails, do not process subsequent ones.
 568     //
 569     if (Token->TransactionStatus != EFI_SUCCESS) {
 570       if (IsListEmpty (&BlkIo2Request->SubtasksQueue) &&
 571           BlkIo2Request->LastSubtaskSubmitted &&
 572           (BlkIo2Request->UnsubmittedSubtaskNum == 0)) {
 573         //
 574         // Remove the BlockIo2 request from the device asynchronous queue.
 575         //
 576         RemoveEntryList (&BlkIo2Request->Link);
 577         FreePool (BlkIo2Request);
 578         gBS->SignalEvent (Token->Event);
 579       }
 580
 581       FreePool (Subtask->CommandPacket->NvmeCmd);
 582       FreePool (Subtask->CommandPacket->NvmeCompletion);
 583       FreePool (Subtask->CommandPacket);
 584       FreePool (Subtask);
 585
 586       continue;
 587     }
 588
 589     Status = Private->Passthru.PassThru (
 590                                  &Private->Passthru,
 591                                  Subtask->NamespaceId,
 592                                  Subtask->CommandPacket,
 593                                  Subtask->Event
 594                                  );
 595     if (Status == EFI_NOT_READY) {
 596       InsertHeadList (&Private->UnsubmittedSubtasks, Link);
 597       BlkIo2Request->UnsubmittedSubtaskNum++;
 598       break;
 599     } else if (EFI_ERROR (Status)) {
 600       Token->TransactionStatus = EFI_DEVICE_ERROR;
 601
 602       if (IsListEmpty (&BlkIo2Request->SubtasksQueue) &&
 603           Subtask->IsLast) {
 604         //
 605         // Remove the BlockIo2 request from the device asynchronous queue.
 606         //
 607         RemoveEntryList (&BlkIo2Request->Link);
 608         FreePool (BlkIo2Request);
 609         gBS->SignalEvent (Token->Event);
 610       }
 611
 612       FreePool (Subtask->CommandPacket->NvmeCmd);
 613       FreePool (Subtask->CommandPacket->NvmeCompletion);
 614       FreePool (Subtask->CommandPacket);
 615       FreePool (Subtask);
 616     } else {
 617       InsertTailList (&BlkIo2Request->SubtasksQueue, Link);
 618       if (Subtask->IsLast) {
 619         BlkIo2Request->LastSubtaskSubmitted = TRUE;
 620       }
 621     }
 622   }
 623
 624   while (Cq->Pt != Private->Pt[QueueId]) {
 625     ASSERT (Cq->Sqid == QueueId);
 626
 627     HasNewItem = TRUE;
 628
 629     //
 630     // Find the command with given Command Id.
 631     //
 632     for (Link = GetFirstNode (&Private->AsyncPassThruQueue);
 633          !IsNull (&Private->AsyncPassThruQueue, Link);
 634          Link = NextLink) {
 635       NextLink = GetNextNode (&Private->AsyncPassThruQueue, Link);
 636       AsyncRequest = NVME_PASS_THRU_ASYNC_REQ_FROM_THIS (Link);
 637       if (AsyncRequest->CommandId == Cq->Cid) {
 638         //
 639         // Copy the Respose Queue entry for this command to the callers
 640         // response buffer.
 641         //
 642         CopyMem (
 643           AsyncRequest->Packet->NvmeCompletion,
 644           Cq,
 645           sizeof(EFI_NVM_EXPRESS_COMPLETION)
 646           );
 647
 648         //
 649         // Free the resources allocated before cmd submission
 650         //
 651         if (AsyncRequest->MapData != NULL) {
 652           PciIo->Unmap (PciIo, AsyncRequest->MapData);
 653         }
 654         if (AsyncRequest->MapMeta != NULL) {
 655           PciIo->Unmap (PciIo, AsyncRequest->MapMeta);
 656         }
 657         if (AsyncRequest->MapPrpList != NULL) {
 658           PciIo->Unmap (PciIo, AsyncRequest->MapPrpList);
 659         }
 660         if (AsyncRequest->PrpListHost != NULL) {
 661           PciIo->FreeBuffer (
 662                    PciIo,
 663                    AsyncRequest->PrpListNo,
 664                    AsyncRequest->PrpListHost
 665                    );
 666         }
 667
 668         RemoveEntryList (Link);
 669         gBS->SignalEvent (AsyncRequest->CallerEvent);
 670         FreePool (AsyncRequest);
 671
 672         //
 673         // Update submission queue head.
 674         //
 675         Private->AsyncSqHead = Cq->Sqhd;
 676         break;
 677       }
 678     }
 679
 680     Private->CqHdbl[QueueId].Cqh++;
 681     if (Private->CqHdbl[QueueId].Cqh > NVME_ASYNC_CCQ_SIZE) {
 682       Private->CqHdbl[QueueId].Cqh = 0;
 683       Private->Pt[QueueId] ^= 1;
 684     }
 685
 686     Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
 687   }
 688
 689   if (HasNewItem) {
 690     Data  = ReadUnaligned32 ((UINT32*)&Private->CqHdbl[QueueId]);
 691     PciIo->Mem.Write (
 692                  PciIo,
 693                  EfiPciIoWidthUint32,
 694                  NVME_BAR,
 695                  NVME_CQHDBL_OFFSET(QueueId, Private->Cap.Dstrd),
 696                  1,
 697                  &Data
 698                  );
 699   }
 700 }
 701
 702 /**
 703   Tests to see if this driver supports a given controller. If a child device is provided,
 704   it further tests to see if this driver supports creating a handle for the specified child device.
 705
 706   This function checks to see if the driver specified by This supports the device specified by
 707   ControllerHandle. Drivers will typically use the device path attached to
 708   ControllerHandle and/or the services from the bus I/O abstraction attached to
 709   ControllerHandle to determine if the driver supports ControllerHandle. This function
 710   may be called many times during platform initialization. In order to reduce boot times, the tests
 711   performed by this function must be very small, and take as little time as possible to execute. This
 712   function must not change the state of any hardware devices, and this function must be aware that the
 713   device specified by ControllerHandle may already be managed by the same driver or a
 714   different driver. This function must match its calls to AllocatePages() with FreePages(),
 715   AllocatePool() with FreePool(), and OpenProtocol() with CloseProtocol().
 716   Since ControllerHandle may have been previously started by the same driver, if a protocol is
 717   already in the opened state, then it must not be closed with CloseProtocol(). This is required
 718   to guarantee the state of ControllerHandle is not modified by this function.
 719
 720   @param[in]  This                 A pointer to the EFI_DRIVER_BINDING_PROTOCOL instance.
 721   @param[in]  ControllerHandle     The handle of the controller to test. This handle
 722                                    must support a protocol interface that supplies
 723                                    an I/O abstraction to the driver.
 724   @param[in]  RemainingDevicePath  A pointer to the remaining portion of a device path.  This
 725                                    parameter is ignored by device drivers, and is optional for bus
 726                                    drivers. For bus drivers, if this parameter is not NULL, then
 727                                    the bus driver must determine if the bus controller specified
 728                                    by ControllerHandle and the child controller specified
 729                                    by RemainingDevicePath are both supported by this
 730                                    bus driver.
 731
 732   @retval EFI_SUCCESS              The device specified by ControllerHandle and
 733                                    RemainingDevicePath is supported by the driver specified by This.
 734   @retval EFI_ALREADY_STARTED      The device specified by ControllerHandle and
 735                                    RemainingDevicePath is already being managed by the driver
 736                                    specified by This.
 737   @retval EFI_ACCESS_DENIED        The device specified by ControllerHandle and
 738                                    RemainingDevicePath is already being managed by a different
 739                                    driver or an application that requires exclusive access.
 740                                    Currently not implemented.
 741   @retval EFI_UNSUPPORTED          The device specified by ControllerHandle and
 742                                    RemainingDevicePath is not supported by the driver specified by This.
 743 **/
 744 EFI_STATUS
 745 EFIAPI
 746 NvmExpressDriverBindingSupported (
 747   IN EFI_DRIVER_BINDING_PROTOCOL  *This,
 748   IN EFI_HANDLE                   Controller,
 749   IN EFI_DEVICE_PATH_PROTOCOL     *RemainingDevicePath
 750   )
 751 {
 752   EFI_STATUS                Status;
 753   EFI_DEV_PATH_PTR          DevicePathNode;
 754   EFI_DEVICE_PATH_PROTOCOL  *ParentDevicePath;
 755   EFI_PCI_IO_PROTOCOL       *PciIo;
 756   UINT8                     ClassCode[3];
 757
 758   //
 759   // Check whether device path is valid
 760   //
 761   if (RemainingDevicePath != NULL) {
 762     //
 763     // Check if RemainingDevicePath is the End of Device Path Node,
 764     // if yes, go on checking other conditions
 765     //
 766     if (!IsDevicePathEnd (RemainingDevicePath)) {
 767       //
 768       // If RemainingDevicePath isn't the End of Device Path Node,
 769       // check its validation
 770       //
 771       DevicePathNode.DevPath = RemainingDevicePath;
 772
 773       if ((DevicePathNode.DevPath->Type    != MESSAGING_DEVICE_PATH) ||
 774           (DevicePathNode.DevPath->SubType != MSG_NVME_NAMESPACE_DP) ||
 775           (DevicePathNodeLength(DevicePathNode.DevPath) != sizeof(NVME_NAMESPACE_DEVICE_PATH))) {
 776          return EFI_UNSUPPORTED;
 777       }
 778     }
 779   }
 780
 781   //
 782   // Open the EFI Device Path protocol needed to perform the supported test
 783   //
 784   Status = gBS->OpenProtocol (
 785                   Controller,
 786                   &gEfiDevicePathProtocolGuid,
 787                   (VOID **) &ParentDevicePath,
 788                   This->DriverBindingHandle,
 789                   Controller,
 790                   EFI_OPEN_PROTOCOL_BY_DRIVER
 791                   );
 792   if (Status == EFI_ALREADY_STARTED) {
 793     return EFI_SUCCESS;
 794   }
 795
 796   if (EFI_ERROR (Status)) {
 797     return Status;
 798   }
 799
 800   //
 801   // Close protocol, don't use device path protocol in the Support() function
 802   //
 803   gBS->CloseProtocol (
 804          Controller,
 805          &gEfiDevicePathProtocolGuid,
 806          This->DriverBindingHandle,
 807          Controller
 808          );
 809
 810   //
 811   // Attempt to Open PCI I/O Protocol
 812   //
 813   Status = gBS->OpenProtocol (
 814                   Controller,
 815                   &gEfiPciIoProtocolGuid,
 816                   (VOID **) &PciIo,
 817                   This->DriverBindingHandle,
 818                   Controller,
 819                   EFI_OPEN_PROTOCOL_BY_DRIVER
 820                   );
 821   if (Status == EFI_ALREADY_STARTED) {
 822     return EFI_SUCCESS;
 823   }
 824
 825   if (EFI_ERROR (Status)) {
 826     return Status;
 827   }
 828
 829   //
 830   // Now further check the PCI header: Base class (offset 0x0B) and Sub Class (offset 0x0A).
 831   // This controller should be a Nvm Express controller.
 832   //
 833   Status = PciIo->Pci.Read (
 834                         PciIo,
 835                         EfiPciIoWidthUint8,
 836                         PCI_CLASSCODE_OFFSET,
 837                         sizeof (ClassCode),
 838                         ClassCode
 839                         );
 840   if (EFI_ERROR (Status)) {
 841     goto Done;
 842   }
 843
 844   //
 845   // Examine Nvm Express controller PCI Configuration table fields
 846   //
 847   if ((ClassCode[0] != PCI_IF_NVMHCI) || (ClassCode[1] != PCI_CLASS_MASS_STORAGE_NVM) || (ClassCode[2] != PCI_CLASS_MASS_STORAGE)) {
 848     Status = EFI_UNSUPPORTED;
 849   }
 850
 851 Done:
 852   gBS->CloseProtocol (
 853          Controller,
 854          &gEfiPciIoProtocolGuid,
 855          This->DriverBindingHandle,
 856          Controller
 857          );
 858
 859   return Status;
 860 }
 861
 862
 863 /**
 864   Starts a device controller or a bus controller.
 865
 866   The Start() function is designed to be invoked from the EFI boot service ConnectController().
 867   As a result, much of the error checking on the parameters to Start() has been moved into this
 868   common boot service. It is legal to call Start() from other locations,
 869   but the following calling restrictions must be followed or the system behavior will not be deterministic.
 870   1. ControllerHandle must be a valid EFI_HANDLE.
 871   2. If RemainingDevicePath is not NULL, then it must be a pointer to a naturally aligned
 872      EFI_DEVICE_PATH_PROTOCOL.
 873   3. Prior to calling Start(), the Supported() function for the driver specified by This must
 874      have been called with the same calling parameters, and Supported() must have returned EFI_SUCCESS.
 875
 876   @param[in]  This                 A pointer to the EFI_DRIVER_BINDING_PROTOCOL instance.
 877   @param[in]  ControllerHandle     The handle of the controller to start. This handle
 878                                    must support a protocol interface that supplies
 879                                    an I/O abstraction to the driver.
 880   @param[in]  RemainingDevicePath  A pointer to the remaining portion of a device path.  This
 881                                    parameter is ignored by device drivers, and is optional for bus
 882                                    drivers. For a bus driver, if this parameter is NULL, then handles
 883                                    for all the children of Controller are created by this driver.
 884                                    If this parameter is not NULL and the first Device Path Node is
 885                                    not the End of Device Path Node, then only the handle for the
 886                                    child device specified by the first Device Path Node of
 887                                    RemainingDevicePath is created by this driver.
 888                                    If the first Device Path Node of RemainingDevicePath is
 889                                    the End of Device Path Node, no child handle is created by this
 890                                    driver.
 891
 892   @retval EFI_SUCCESS              The device was started.
 893   @retval EFI_DEVICE_ERROR         The device could not be started due to a device error.Currently not implemented.
 894   @retval EFI_OUT_OF_RESOURCES     The request could not be completed due to a lack of resources.
 895   @retval Others                   The driver failded to start the device.
 896
 897 **/
 898 EFI_STATUS
 899 EFIAPI
 900 NvmExpressDriverBindingStart (
 901   IN EFI_DRIVER_BINDING_PROTOCOL  *This,
 902   IN EFI_HANDLE                   Controller,
 903   IN EFI_DEVICE_PATH_PROTOCOL     *RemainingDevicePath
 904   )
 905 {
 906   EFI_STATUS                          Status;
 907   EFI_PCI_IO_PROTOCOL                 *PciIo;
 908   NVME_CONTROLLER_PRIVATE_DATA        *Private;
 909   EFI_DEVICE_PATH_PROTOCOL            *ParentDevicePath;
 910   UINT32                              NamespaceId;
 911   EFI_PHYSICAL_ADDRESS                MappedAddr;
 912   UINTN                               Bytes;
 913   EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL  *Passthru;
 914
 915   DEBUG ((EFI_D_INFO, "NvmExpressDriverBindingStart: start\n"));
 916
 917   Private          = NULL;
 918   Passthru         = NULL;
 919   ParentDevicePath = NULL;
 920
 921   Status = gBS->OpenProtocol (
 922                   Controller,
 923                   &gEfiDevicePathProtocolGuid,
 924                   (VOID **) &ParentDevicePath,
 925                   This->DriverBindingHandle,
 926                   Controller,
 927                   EFI_OPEN_PROTOCOL_BY_DRIVER
 928                   );
 929   if ((EFI_ERROR (Status)) && (Status != EFI_ALREADY_STARTED)) {
 930     return Status;
 931   }
 932
 933   Status = gBS->OpenProtocol (
 934                   Controller,
 935                   &gEfiPciIoProtocolGuid,
 936                   (VOID **) &PciIo,
 937                   This->DriverBindingHandle,
 938                   Controller,
 939                   EFI_OPEN_PROTOCOL_BY_DRIVER
 940                   );
 941
 942   if (EFI_ERROR (Status) && (Status != EFI_ALREADY_STARTED)) {
 943     return Status;
 944   }
 945
 946   //
 947   // Check EFI_ALREADY_STARTED to reuse the original NVME_CONTROLLER_PRIVATE_DATA.
 948   //
 949   if (Status != EFI_ALREADY_STARTED) {
 950     Private = AllocateZeroPool (sizeof (NVME_CONTROLLER_PRIVATE_DATA));
 951
 952     if (Private == NULL) {
 953       DEBUG ((EFI_D_ERROR, "NvmExpressDriverBindingStart: allocating pool for Nvme Private Data failed!\n"));
 954       Status = EFI_OUT_OF_RESOURCES;
 955       goto Exit;
 956     }
 957
 958     //
 959     // 6 x 4kB aligned buffers will be carved out of this buffer.
 960     // 1st 4kB boundary is the start of the admin submission queue.
 961     // 2nd 4kB boundary is the start of the admin completion queue.
 962     // 3rd 4kB boundary is the start of I/O submission queue #1.
 963     // 4th 4kB boundary is the start of I/O completion queue #1.
 964     // 5th 4kB boundary is the start of I/O submission queue #2.
 965     // 6th 4kB boundary is the start of I/O completion queue #2.
 966     //
 967     // Allocate 6 pages of memory, then map it for bus master read and write.
 968     //
 969     Status = PciIo->AllocateBuffer (
 970                       PciIo,
 971                       AllocateAnyPages,
 972                       EfiBootServicesData,
 973                       6,
 974                       (VOID**)&Private->Buffer,
 975                       0
 976                       );
 977     if (EFI_ERROR (Status)) {
 978       goto Exit;
 979     }
 980
 981     Bytes = EFI_PAGES_TO_SIZE (6);
 982     Status = PciIo->Map (
 983                       PciIo,
 984                       EfiPciIoOperationBusMasterCommonBuffer,
 985                       Private->Buffer,
 986                       &Bytes,
 987                       &MappedAddr,
 988                       &Private->Mapping
 989                       );
 990
 991     if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (6))) {
 992       goto Exit;
 993     }
 994
 995     Private->BufferPciAddr = (UINT8 *)(UINTN)MappedAddr;
 996
 997     Private->Signature = NVME_CONTROLLER_PRIVATE_DATA_SIGNATURE;
 998     Private->ControllerHandle          = Controller;
 999     Private->ImageHandle               = This->DriverBindingHandle;
1000     Private->DriverBindingHandle       = This->DriverBindingHandle;
1001     Private->PciIo                     = PciIo;
1002     Private->ParentDevicePath          = ParentDevicePath;
1003     Private->Passthru.Mode             = &Private->PassThruMode;
1004     Private->Passthru.PassThru         = NvmExpressPassThru;
1005     Private->Passthru.GetNextNamespace = NvmExpressGetNextNamespace;
1006     Private->Passthru.BuildDevicePath  = NvmExpressBuildDevicePath;
1007     Private->Passthru.GetNamespace     = NvmExpressGetNamespace;
1008     CopyMem (&Private->PassThruMode, &gEfiNvmExpressPassThruMode, sizeof (EFI_NVM_EXPRESS_PASS_THRU_MODE));
1009     InitializeListHead (&Private->AsyncPassThruQueue);
1010     InitializeListHead (&Private->UnsubmittedSubtasks);
1011
1012     Status = NvmeControllerInit (Private);
1013     if (EFI_ERROR(Status)) {
1014       goto Exit;
1015     }
1016
1017     //
1018     // Start the asynchronous I/O completion monitor
1019     //
1020     Status = gBS->CreateEvent (
1021                     EVT_TIMER | EVT_NOTIFY_SIGNAL,
1022                     TPL_NOTIFY,
1023                     ProcessAsyncTaskList,
1024                     Private,
1025                     &Private->TimerEvent
1026                     );
1027     if (EFI_ERROR (Status)) {
1028       goto Exit;
1029     }
1030
1031     Status = gBS->SetTimer (
1032                     Private->TimerEvent,
1033                     TimerPeriodic,
1034                     NVME_HC_ASYNC_TIMER
1035                     );
1036     if (EFI_ERROR (Status)) {
1037       goto Exit;
1038     }
1039
1040     Status = gBS->InstallMultipleProtocolInterfaces (
1041                     &Controller,
1042                     &gEfiNvmExpressPassThruProtocolGuid,
1043                     &Private->Passthru,
1044                     NULL
1045                     );
1046     if (EFI_ERROR (Status)) {
1047       goto Exit;
1048     }
1049   } else {
1050     Status = gBS->OpenProtocol (
1051                     Controller,
1052                     &gEfiNvmExpressPassThruProtocolGuid,
1053                     (VOID **) &Passthru,
1054                     This->DriverBindingHandle,
1055                     Controller,
1056                     EFI_OPEN_PROTOCOL_GET_PROTOCOL
1057                     );
1058     if (EFI_ERROR (Status)) {
1059       goto Exit;
1060     }
1061
1062     Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (Passthru);
1063   }
1064
1065   if (RemainingDevicePath == NULL) {
1066     //
1067     // Enumerate all NVME namespaces in the controller
1068     //
1069     Status = DiscoverAllNamespaces (
1070                Private
1071                );
1072
1073   } else if (!IsDevicePathEnd (RemainingDevicePath)) {
1074     //
1075     // Enumerate the specified NVME namespace
1076     //
1077     Status = Private->Passthru.GetNamespace (
1078                                  &Private->Passthru,
1079                                  RemainingDevicePath,
1080                                  &NamespaceId
1081                                  );
1082
1083     if (!EFI_ERROR (Status)) {
1084       Status = EnumerateNvmeDevNamespace (
1085                  Private,
1086                  NamespaceId
1087                  );
1088     }
1089   }
1090
1091   DEBUG ((EFI_D_INFO, "NvmExpressDriverBindingStart: end successfully\n"));
1092   return EFI_SUCCESS;
1093
1094 Exit:
1095   if ((Private != NULL) && (Private->Mapping != NULL)) {
1096     PciIo->Unmap (PciIo, Private->Mapping);
1097   }
1098
1099   if ((Private != NULL) && (Private->Buffer != NULL)) {
1100     PciIo->FreeBuffer (PciIo, 6, Private->Buffer);
1101   }
1102
1103   if ((Private != NULL) && (Private->ControllerData != NULL)) {
1104     FreePool (Private->ControllerData);
1105   }
1106
1107   if (Private != NULL) {
1108     if (Private->TimerEvent != NULL) {
1109       gBS->CloseEvent (Private->TimerEvent);
1110     }
1111
1112     FreePool (Private);
1113   }
1114
1115   gBS->CloseProtocol (
1116          Controller,
1117          &gEfiPciIoProtocolGuid,
1118          This->DriverBindingHandle,
1119          Controller
1120          );
1121
1122   gBS->CloseProtocol (
1123          Controller,
1124          &gEfiDevicePathProtocolGuid,
1125          This->DriverBindingHandle,
1126          Controller
1127          );
1128
1129   DEBUG ((EFI_D_INFO, "NvmExpressDriverBindingStart: end with %r\n", Status));
1130
1131   return Status;
1132 }
1133
1134
1135 /**
1136   Stops a device controller or a bus controller.
1137
1138   The Stop() function is designed to be invoked from the EFI boot service DisconnectController().
1139   As a result, much of the error checking on the parameters to Stop() has been moved
1140   into this common boot service. It is legal to call Stop() from other locations,
1141   but the following calling restrictions must be followed or the system behavior will not be deterministic.
1142   1. ControllerHandle must be a valid EFI_HANDLE that was used on a previous call to this
1143      same driver's Start() function.
1144   2. The first NumberOfChildren handles of ChildHandleBuffer must all be a valid
1145      EFI_HANDLE. In addition, all of these handles must have been created in this driver's
1146      Start() function, and the Start() function must have called OpenProtocol() on
1147      ControllerHandle with an Attribute of EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER.
1148
1149   @param[in]  This              A pointer to the EFI_DRIVER_BINDING_PROTOCOL instance.
1150   @param[in]  ControllerHandle  A handle to the device being stopped. The handle must
1151                                 support a bus specific I/O protocol for the driver
1152                                 to use to stop the device.
1153   @param[in]  NumberOfChildren  The number of child device handles in ChildHandleBuffer.
1154   @param[in]  ChildHandleBuffer An array of child handles to be freed. May be NULL
1155                                 if NumberOfChildren is 0.
1156
1157   @retval EFI_SUCCESS           The device was stopped.
1158   @retval EFI_DEVICE_ERROR      The device could not be stopped due to a device error.
1159
1160 **/
1161 EFI_STATUS
1162 EFIAPI
1163 NvmExpressDriverBindingStop (
1164   IN  EFI_DRIVER_BINDING_PROTOCOL     *This,
1165   IN  EFI_HANDLE                      Controller,
1166   IN  UINTN                           NumberOfChildren,
1167   IN  EFI_HANDLE                      *ChildHandleBuffer
1168   )
1169 {
1170   EFI_STATUS                          Status;
1171   BOOLEAN                             AllChildrenStopped;
1172   UINTN                               Index;
1173   NVME_CONTROLLER_PRIVATE_DATA        *Private;
1174   EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL  *PassThru;
1175   BOOLEAN                             IsEmpty;
1176   EFI_TPL                             OldTpl;
1177
1178   if (NumberOfChildren == 0) {
1179     Status = gBS->OpenProtocol (
1180                     Controller,
1181                     &gEfiNvmExpressPassThruProtocolGuid,
1182                     (VOID **) &PassThru,
1183                     This->DriverBindingHandle,
1184                     Controller,
1185                     EFI_OPEN_PROTOCOL_GET_PROTOCOL
1186                     );
1187
1188     if (!EFI_ERROR (Status)) {
1189       Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (PassThru);
1190
1191       //
1192       // Wait for the asynchronous PassThru queue to become empty.
1193       //
1194       while (TRUE) {
1195         OldTpl  = gBS->RaiseTPL (TPL_NOTIFY);
1196         IsEmpty = IsListEmpty (&Private->AsyncPassThruQueue) &&
1197                   IsListEmpty (&Private->UnsubmittedSubtasks);
1198         gBS->RestoreTPL (OldTpl);
1199
1200         if (IsEmpty) {
1201           break;
1202         }
1203
1204         gBS->Stall (100);
1205       }
1206
1207       gBS->UninstallMultipleProtocolInterfaces (
1208             Controller,
1209             &gEfiNvmExpressPassThruProtocolGuid,
1210             PassThru,
1211             NULL
1212             );
1213
1214       if (Private->TimerEvent != NULL) {
1215         gBS->CloseEvent (Private->TimerEvent);
1216       }
1217
1218       if (Private->Mapping != NULL) {
1219         Private->PciIo->Unmap (Private->PciIo, Private->Mapping);
1220       }
1221
1222       if (Private->Buffer != NULL) {
1223         Private->PciIo->FreeBuffer (Private->PciIo, 6, Private->Buffer);
1224       }
1225
1226       FreePool (Private->ControllerData);
1227       FreePool (Private);
1228     }
1229
1230     gBS->CloseProtocol (
1231           Controller,
1232           &gEfiPciIoProtocolGuid,
1233           This->DriverBindingHandle,
1234           Controller
1235           );
1236     gBS->CloseProtocol (
1237           Controller,
1238           &gEfiDevicePathProtocolGuid,
1239           This->DriverBindingHandle,
1240           Controller
1241           );
1242     return EFI_SUCCESS;
1243   }
1244
1245   AllChildrenStopped = TRUE;
1246
1247   for (Index = 0; Index < NumberOfChildren; Index++) {
1248     Status = UnregisterNvmeNamespace (This, Controller, ChildHandleBuffer[Index]);
1249     if (EFI_ERROR (Status)) {
1250       AllChildrenStopped = FALSE;
1251     }
1252   }
1253
1254   if (!AllChildrenStopped) {
1255     return EFI_DEVICE_ERROR;
1256   }
1257
1258   return EFI_SUCCESS;
1259 }
1260
1261 /**
1262   This is the unload handle for the NVM Express driver.
1263
1264   Disconnect the driver specified by ImageHandle from the NVMe device in the handle database.
1265   Uninstall all the protocols installed in the driver.
1266
1267   @param[in]  ImageHandle       The drivers' driver image.
1268
1269   @retval EFI_SUCCESS           The image is unloaded.
1270   @retval Others                Failed to unload the image.
1271
1272 **/
1273 EFI_STATUS
1274 EFIAPI
1275 NvmExpressUnload (
1276   IN EFI_HANDLE             ImageHandle
1277   )
1278 {
1279   EFI_STATUS                        Status;
1280   EFI_HANDLE                        *DeviceHandleBuffer;
1281   UINTN                             DeviceHandleCount;
1282   UINTN                             Index;
1283   EFI_COMPONENT_NAME_PROTOCOL       *ComponentName;
1284   EFI_COMPONENT_NAME2_PROTOCOL      *ComponentName2;
1285
1286   //
1287   // Get the list of the device handles managed by this driver.
1288   // If there is an error getting the list, then means the driver
1289   // doesn't manage any device. At this way, we would only close
1290   // those protocols installed at image handle.
1291   //
1292   DeviceHandleBuffer = NULL;
1293   Status = gBS->LocateHandleBuffer (
1294                   ByProtocol,
1295                   &gEfiNvmExpressPassThruProtocolGuid,
1296                   NULL,
1297                   &DeviceHandleCount,
1298                   &DeviceHandleBuffer
1299                   );
1300
1301   if (!EFI_ERROR (Status)) {
1302     //
1303     // Disconnect the driver specified by ImageHandle from all
1304     // the devices in the handle database.
1305     //
1306     for (Index = 0; Index < DeviceHandleCount; Index++) {
1307       Status = gBS->DisconnectController (
1308                       DeviceHandleBuffer[Index],
1309                       ImageHandle,
1310                       NULL
1311                       );
1312       if (EFI_ERROR (Status)) {
1313         goto EXIT;
1314       }
1315     }
1316   }
1317
1318   //
1319   // Uninstall all the protocols installed in the driver entry point
1320   //
1321   Status = gBS->UninstallMultipleProtocolInterfaces (
1322                   ImageHandle,
1323                   &gEfiDriverBindingProtocolGuid,
1324                   &gNvmExpressDriverBinding,
1325                   &gEfiDriverSupportedEfiVersionProtocolGuid,
1326                   &gNvmExpressDriverSupportedEfiVersion,
1327                   NULL
1328                   );
1329
1330   if (EFI_ERROR (Status)) {
1331     goto EXIT;
1332   }
1333
1334   //
1335   // Note we have to one by one uninstall the following protocols.
1336   // It's because some of them are optionally installed based on
1337   // the following PCD settings.
1338   //   gEfiMdePkgTokenSpaceGuid.PcdDriverDiagnosticsDisable
1339   //   gEfiMdePkgTokenSpaceGuid.PcdComponentNameDisable
1340   //   gEfiMdePkgTokenSpaceGuid.PcdDriverDiagnostics2Disable
1341   //   gEfiMdePkgTokenSpaceGuid.PcdComponentName2Disable
1342   //
1343   Status = gBS->HandleProtocol (
1344                   ImageHandle,
1345                   &gEfiComponentNameProtocolGuid,
1346                   (VOID **) &ComponentName
1347                   );
1348   if (!EFI_ERROR (Status)) {
1349     gBS->UninstallProtocolInterface (
1350            ImageHandle,
1351            &gEfiComponentNameProtocolGuid,
1352            ComponentName
1353            );
1354   }
1355
1356   Status = gBS->HandleProtocol (
1357                   ImageHandle,
1358                   &gEfiComponentName2ProtocolGuid,
1359                   (VOID **) &ComponentName2
1360                   );
1361   if (!EFI_ERROR (Status)) {
1362     gBS->UninstallProtocolInterface (
1363            ImageHandle,
1364            &gEfiComponentName2ProtocolGuid,
1365            ComponentName2
1366            );
1367   }
1368
1369   Status = EFI_SUCCESS;
1370
1371 EXIT:
1372   //
1373   // Free the buffer containing the list of handles from the handle database
1374   //
1375   if (DeviceHandleBuffer != NULL) {
1376     gBS->FreePool (DeviceHandleBuffer);
1377   }
1378   return Status;
1379 }
1380
1381 /**
1382   The entry point for Nvm Express driver, used to install Nvm Express driver on the ImageHandle.
1383
1384   @param  ImageHandle   The firmware allocated handle for this driver image.
1385   @param  SystemTable   Pointer to the EFI system table.
1386
1387   @retval EFI_SUCCESS   Driver loaded.
1388   @retval other         Driver not loaded.
1389
1390 **/
1391 EFI_STATUS
1392 EFIAPI
1393 NvmExpressDriverEntry (
1394   IN EFI_HANDLE        ImageHandle,
1395   IN EFI_SYSTEM_TABLE  *SystemTable
1396   )
1397 {
1398   EFI_STATUS              Status;
1399
1400   Status = EfiLibInstallDriverBindingComponentName2 (
1401              ImageHandle,
1402              SystemTable,
1403              &gNvmExpressDriverBinding,
1404              ImageHandle,
1405              &gNvmExpressComponentName,
1406              &gNvmExpressComponentName2
1407              );
1408   ASSERT_EFI_ERROR (Status);
1409
1410   //
1411   // Install EFI Driver Supported EFI Version Protocol required for
1412   // EFI drivers that are on PCI and other plug in cards.
1413   //
1414   gNvmExpressDriverSupportedEfiVersion.FirmwareVersion = 0x00020028;
1415   Status = gBS->InstallMultipleProtocolInterfaces (
1416                   &ImageHandle,
1417                   &gEfiDriverSupportedEfiVersionProtocolGuid,
1418                   &gNvmExpressDriverSupportedEfiVersion,
1419                   NULL
1420                   );
1421   ASSERT_EFI_ERROR (Status);
1422   return Status;
1423 }