MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.c

   1 /** @file
   2   NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows
   3   NVM Express specification.
   4
   5   Copyright (c) 2013 - 2017, Intel Corporation. All rights reserved.<BR>
   6   SPDX-License-Identifier: BSD-2-Clause-Patent
   7
   8 **/
   9
  10 #include "NvmExpress.h"
  11
  12 //
  13 // NVM Express Driver Binding Protocol Instance
  14 //
  15 EFI_DRIVER_BINDING_PROTOCOL gNvmExpressDriverBinding = {
  16   NvmExpressDriverBindingSupported,
  17   NvmExpressDriverBindingStart,
  18   NvmExpressDriverBindingStop,
  19   0x10,
  20   NULL,
  21   NULL
  22 };
  23
  24 //
  25 // NVM Express EFI Driver Supported EFI Version Protocol Instance
  26 //
  27 EFI_DRIVER_SUPPORTED_EFI_VERSION_PROTOCOL gNvmExpressDriverSupportedEfiVersion = {
  28   sizeof (EFI_DRIVER_SUPPORTED_EFI_VERSION_PROTOCOL), // Size of Protocol structure.
  29   0                                                   // Version number to be filled at start up.
  30 };
  31
  32 //
  33 // Template for NVM Express Pass Thru Mode data structure.
  34 //
  35 GLOBAL_REMOVE_IF_UNREFERENCED EFI_NVM_EXPRESS_PASS_THRU_MODE gEfiNvmExpressPassThruMode = {
  36   EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL   |
  37   EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL    |
  38   EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_NONBLOCKIO |
  39   EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_CMD_SET_NVM,
  40   sizeof (UINTN),
  41   0x10100
  42 };
  43
  44 /**
  45   Check if the specified Nvm Express device namespace is active, and create child handles
  46   for them with BlockIo and DiskInfo protocol instances.
  47
  48   @param[in] Private         The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
  49   @param[in] NamespaceId     The NVM Express namespace ID  for which a device path node is to be
  50                              allocated and built. Caller must set the NamespaceId to zero if the
  51                              device path node will contain a valid UUID.
  52
  53   @retval EFI_SUCCESS        All the namespaces in the device are successfully enumerated.
  54   @return Others             Some error occurs when enumerating the namespaces.
  55
  56 **/
  57 EFI_STATUS
  58 EnumerateNvmeDevNamespace (
  59   IN NVME_CONTROLLER_PRIVATE_DATA       *Private,
  60   UINT32                                NamespaceId
  61   )
  62 {
  63   NVME_ADMIN_NAMESPACE_DATA             *NamespaceData;
  64   EFI_DEVICE_PATH_PROTOCOL              *NewDevicePathNode;
  65   EFI_DEVICE_PATH_PROTOCOL              *DevicePath;
  66   EFI_HANDLE                            DeviceHandle;
  67   EFI_DEVICE_PATH_PROTOCOL              *ParentDevicePath;
  68   EFI_DEVICE_PATH_PROTOCOL              *RemainingDevicePath;
  69   NVME_DEVICE_PRIVATE_DATA              *Device;
  70   EFI_STATUS                            Status;
  71   UINT32                                Lbads;
  72   UINT32                                Flbas;
  73   UINT32                                LbaFmtIdx;
  74   UINT8                                 Sn[21];
  75   UINT8                                 Mn[41];
  76   VOID                                  *DummyInterface;
  77
  78   NewDevicePathNode = NULL;
  79   DevicePath        = NULL;
  80   Device            = NULL;
  81
  82   //
  83   // Allocate a buffer for Identify Namespace data
  84   //
  85   NamespaceData = AllocateZeroPool(sizeof (NVME_ADMIN_NAMESPACE_DATA));
  86   if(NamespaceData == NULL) {
  87     return EFI_OUT_OF_RESOURCES;
  88   }
  89
  90   ParentDevicePath = Private->ParentDevicePath;
  91   //
  92   // Identify Namespace
  93   //
  94   Status = NvmeIdentifyNamespace (
  95              Private,
  96              NamespaceId,
  97              (VOID *)NamespaceData
  98              );
  99   if (EFI_ERROR(Status)) {
 100     goto Exit;
 101   }
 102   //
 103   // Validate Namespace
 104   //
 105   if (NamespaceData->Ncap == 0) {
 106     Status = EFI_DEVICE_ERROR;
 107   } else {
 108     //
 109     // allocate device private data for each discovered namespace
 110     //
 111     Device = AllocateZeroPool(sizeof(NVME_DEVICE_PRIVATE_DATA));
 112     if (Device == NULL) {
 113       Status = EFI_OUT_OF_RESOURCES;
 114       goto Exit;
 115     }
 116
 117     //
 118     // Initialize SSD namespace instance data
 119     //
 120     Device->Signature           = NVME_DEVICE_PRIVATE_DATA_SIGNATURE;
 121     Device->NamespaceId         = NamespaceId;
 122     Device->NamespaceUuid       = NamespaceData->Eui64;
 123
 124     Device->ControllerHandle    = Private->ControllerHandle;
 125     Device->DriverBindingHandle = Private->DriverBindingHandle;
 126     Device->Controller          = Private;
 127
 128     //
 129     // Build BlockIo media structure
 130     //
 131     Device->Media.MediaId        = 0;
 132     Device->Media.RemovableMedia = FALSE;
 133     Device->Media.MediaPresent   = TRUE;
 134     Device->Media.LogicalPartition = FALSE;
 135     Device->Media.ReadOnly       = FALSE;
 136     Device->Media.WriteCaching   = FALSE;
 137     Device->Media.IoAlign        = Private->PassThruMode.IoAlign;
 138
 139     Flbas     = NamespaceData->Flbas;
 140     LbaFmtIdx = Flbas & 0xF;
 141     Lbads     = NamespaceData->LbaFormat[LbaFmtIdx].Lbads;
 142     Device->Media.BlockSize = (UINT32)1 << Lbads;
 143
 144     Device->Media.LastBlock                     = NamespaceData->Nsze - 1;
 145     Device->Media.LogicalBlocksPerPhysicalBlock = 1;
 146     Device->Media.LowestAlignedLba              = 1;
 147
 148     //
 149     // Create BlockIo Protocol instance
 150     //
 151     Device->BlockIo.Revision     = EFI_BLOCK_IO_PROTOCOL_REVISION2;
 152     Device->BlockIo.Media        = &Device->Media;
 153     Device->BlockIo.Reset        = NvmeBlockIoReset;
 154     Device->BlockIo.ReadBlocks   = NvmeBlockIoReadBlocks;
 155     Device->BlockIo.WriteBlocks  = NvmeBlockIoWriteBlocks;
 156     Device->BlockIo.FlushBlocks  = NvmeBlockIoFlushBlocks;
 157
 158     //
 159     // Create BlockIo2 Protocol instance
 160     //
 161     Device->BlockIo2.Media          = &Device->Media;
 162     Device->BlockIo2.Reset          = NvmeBlockIoResetEx;
 163     Device->BlockIo2.ReadBlocksEx   = NvmeBlockIoReadBlocksEx;
 164     Device->BlockIo2.WriteBlocksEx  = NvmeBlockIoWriteBlocksEx;
 165     Device->BlockIo2.FlushBlocksEx  = NvmeBlockIoFlushBlocksEx;
 166     InitializeListHead (&Device->AsyncQueue);
 167
 168     //
 169     // Create StorageSecurityProtocol Instance
 170     //
 171     Device->StorageSecurity.ReceiveData = NvmeStorageSecurityReceiveData;
 172     Device->StorageSecurity.SendData    = NvmeStorageSecuritySendData;
 173
 174     //
 175     // Create DiskInfo Protocol instance
 176     //
 177     CopyMem (&Device->NamespaceData, NamespaceData, sizeof (NVME_ADMIN_NAMESPACE_DATA));
 178     InitializeDiskInfo (Device);
 179
 180     //
 181     // Create a Nvm Express Namespace Device Path Node
 182     //
 183     Status = Private->Passthru.BuildDevicePath (
 184                                  &Private->Passthru,
 185                                  Device->NamespaceId,
 186                                  &NewDevicePathNode
 187                                  );
 188
 189     if (EFI_ERROR(Status)) {
 190       goto Exit;
 191     }
 192
 193     //
 194     // Append the SSD node to the controller's device path
 195     //
 196     DevicePath = AppendDevicePathNode (ParentDevicePath, NewDevicePathNode);
 197     if (DevicePath == NULL) {
 198       Status = EFI_OUT_OF_RESOURCES;
 199       goto Exit;
 200     }
 201
 202     DeviceHandle = NULL;
 203     RemainingDevicePath = DevicePath;
 204     Status = gBS->LocateDevicePath (&gEfiDevicePathProtocolGuid, &RemainingDevicePath, &DeviceHandle);
 205     if (!EFI_ERROR (Status) && (DeviceHandle != NULL) && IsDevicePathEnd(RemainingDevicePath)) {
 206       Status = EFI_ALREADY_STARTED;
 207       FreePool (DevicePath);
 208       goto Exit;
 209     }
 210
 211     Device->DevicePath = DevicePath;
 212
 213     //
 214     // Make sure the handle is NULL so we create a new handle
 215     //
 216     Device->DeviceHandle = NULL;
 217
 218     Status = gBS->InstallMultipleProtocolInterfaces (
 219                     &Device->DeviceHandle,
 220                     &gEfiDevicePathProtocolGuid,
 221                     Device->DevicePath,
 222                     &gEfiBlockIoProtocolGuid,
 223                     &Device->BlockIo,
 224                     &gEfiBlockIo2ProtocolGuid,
 225                     &Device->BlockIo2,
 226                     &gEfiDiskInfoProtocolGuid,
 227                     &Device->DiskInfo,
 228                     NULL
 229                     );
 230
 231     if(EFI_ERROR(Status)) {
 232       goto Exit;
 233     }
 234
 235     //
 236     // Check if the NVMe controller supports the Security Send and Security Receive commands
 237     //
 238     if ((Private->ControllerData->Oacs & SECURITY_SEND_RECEIVE_SUPPORTED) != 0) {
 239       Status = gBS->InstallProtocolInterface (
 240                       &Device->DeviceHandle,
 241                       &gEfiStorageSecurityCommandProtocolGuid,
 242                       EFI_NATIVE_INTERFACE,
 243                       &Device->StorageSecurity
 244                       );
 245       if(EFI_ERROR(Status)) {
 246         gBS->UninstallMultipleProtocolInterfaces (
 247                &Device->DeviceHandle,
 248                &gEfiDevicePathProtocolGuid,
 249                Device->DevicePath,
 250                &gEfiBlockIoProtocolGuid,
 251                &Device->BlockIo,
 252                &gEfiBlockIo2ProtocolGuid,
 253                &Device->BlockIo2,
 254                &gEfiDiskInfoProtocolGuid,
 255                &Device->DiskInfo,
 256                NULL
 257                );
 258         goto Exit;
 259       }
 260     }
 261
 262     gBS->OpenProtocol (
 263            Private->ControllerHandle,
 264            &gEfiNvmExpressPassThruProtocolGuid,
 265            (VOID **) &DummyInterface,
 266            Private->DriverBindingHandle,
 267            Device->DeviceHandle,
 268            EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER
 269            );
 270
 271     //
 272     // Dump NvmExpress Identify Namespace Data
 273     //
 274     DEBUG ((EFI_D_INFO, " == NVME IDENTIFY NAMESPACE [%d] DATA ==\n", NamespaceId));
 275     DEBUG ((EFI_D_INFO, "    NSZE        : 0x%x\n", NamespaceData->Nsze));
 276     DEBUG ((EFI_D_INFO, "    NCAP        : 0x%x\n", NamespaceData->Ncap));
 277     DEBUG ((EFI_D_INFO, "    NUSE        : 0x%x\n", NamespaceData->Nuse));
 278     DEBUG ((EFI_D_INFO, "    LBAF0.LBADS : 0x%x\n", (NamespaceData->LbaFormat[0].Lbads)));
 279
 280     //
 281     // Build controller name for Component Name (2) protocol.
 282     //
 283     CopyMem (Sn, Private->ControllerData->Sn, sizeof (Private->ControllerData->Sn));
 284     Sn[20] = 0;
 285     CopyMem (Mn, Private->ControllerData->Mn, sizeof (Private->ControllerData->Mn));
 286     Mn[40] = 0;
 287     UnicodeSPrintAsciiFormat (Device->ModelName, sizeof (Device->ModelName), "%a-%a-%x", Sn, Mn, NamespaceData->Eui64);
 288
 289     AddUnicodeString2 (
 290       "eng",
 291       gNvmExpressComponentName.SupportedLanguages,
 292       &Device->ControllerNameTable,
 293       Device->ModelName,
 294       TRUE
 295       );
 296
 297     AddUnicodeString2 (
 298       "en",
 299       gNvmExpressComponentName2.SupportedLanguages,
 300       &Device->ControllerNameTable,
 301       Device->ModelName,
 302       FALSE
 303       );
 304   }
 305
 306 Exit:
 307   if(NamespaceData != NULL) {
 308     FreePool (NamespaceData);
 309   }
 310
 311   if (NewDevicePathNode != NULL) {
 312     FreePool (NewDevicePathNode);
 313   }
 314
 315   if(EFI_ERROR(Status) && (Device != NULL) && (Device->DevicePath != NULL)) {
 316     FreePool (Device->DevicePath);
 317   }
 318   if(EFI_ERROR(Status) && (Device != NULL)) {
 319     FreePool (Device);
 320   }
 321   return Status;
 322 }
 323
 324 /**
 325   Discover all Nvm Express device namespaces, and create child handles for them with BlockIo
 326   and DiskInfo protocol instances.
 327
 328   @param[in] Private         The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
 329
 330   @retval EFI_SUCCESS        All the namespaces in the device are successfully enumerated.
 331   @return Others             Some error occurs when enumerating the namespaces.
 332
 333 **/
 334 EFI_STATUS
 335 DiscoverAllNamespaces (
 336   IN NVME_CONTROLLER_PRIVATE_DATA       *Private
 337   )
 338 {
 339   EFI_STATUS                            Status;
 340   UINT32                                NamespaceId;
 341   EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL    *Passthru;
 342
 343   NamespaceId   = 0xFFFFFFFF;
 344   Passthru      = &Private->Passthru;
 345
 346   while (TRUE) {
 347     Status = Passthru->GetNextNamespace (
 348                          Passthru,
 349                          (UINT32 *)&NamespaceId
 350                          );
 351
 352     if (EFI_ERROR (Status)) {
 353       break;
 354     }
 355
 356     Status = EnumerateNvmeDevNamespace (
 357                Private,
 358                NamespaceId
 359                );
 360
 361     if (EFI_ERROR(Status)) {
 362       continue;
 363     }
 364   }
 365
 366   return EFI_SUCCESS;
 367 }
 368
 369 /**
 370   Unregisters a Nvm Express device namespace.
 371
 372   This function removes the protocols installed on the controller handle and
 373   frees the resources allocated for the namespace.
 374
 375   @param  This                  The pointer to EFI_DRIVER_BINDING_PROTOCOL instance.
 376   @param  Controller            The controller handle of the namespace.
 377   @param  Handle                The child handle.
 378
 379   @retval EFI_SUCCESS           The namespace is successfully unregistered.
 380   @return Others                Some error occurs when unregistering the namespace.
 381
 382 **/
 383 EFI_STATUS
 384 UnregisterNvmeNamespace (
 385   IN  EFI_DRIVER_BINDING_PROTOCOL    *This,
 386   IN  EFI_HANDLE                     Controller,
 387   IN  EFI_HANDLE                     Handle
 388   )
 389 {
 390   EFI_STATUS                               Status;
 391   EFI_BLOCK_IO_PROTOCOL                    *BlockIo;
 392   NVME_DEVICE_PRIVATE_DATA                 *Device;
 393   EFI_STORAGE_SECURITY_COMMAND_PROTOCOL    *StorageSecurity;
 394   BOOLEAN                                  IsEmpty;
 395   EFI_TPL                                  OldTpl;
 396   VOID                                     *DummyInterface;
 397
 398   BlockIo = NULL;
 399
 400   Status = gBS->OpenProtocol (
 401                   Handle,
 402                   &gEfiBlockIoProtocolGuid,
 403                   (VOID **) &BlockIo,
 404                   This->DriverBindingHandle,
 405                   Controller,
 406                   EFI_OPEN_PROTOCOL_GET_PROTOCOL
 407                   );
 408   if (EFI_ERROR (Status)) {
 409     return Status;
 410   }
 411
 412   Device  = NVME_DEVICE_PRIVATE_DATA_FROM_BLOCK_IO (BlockIo);
 413
 414   //
 415   // Wait for the device's asynchronous I/O queue to become empty.
 416   //
 417   while (TRUE) {
 418     OldTpl  = gBS->RaiseTPL (TPL_NOTIFY);
 419     IsEmpty = IsListEmpty (&Device->AsyncQueue);
 420     gBS->RestoreTPL (OldTpl);
 421
 422     if (IsEmpty) {
 423       break;
 424     }
 425
 426     gBS->Stall (100);
 427   }
 428
 429   //
 430   // Close the child handle
 431   //
 432   gBS->CloseProtocol (
 433          Controller,
 434          &gEfiNvmExpressPassThruProtocolGuid,
 435          This->DriverBindingHandle,
 436          Handle
 437          );
 438
 439   //
 440   // The Nvm Express driver installs the BlockIo and DiskInfo in the DriverBindingStart().
 441   // Here should uninstall both of them.
 442   //
 443   Status = gBS->UninstallMultipleProtocolInterfaces (
 444                   Handle,
 445                   &gEfiDevicePathProtocolGuid,
 446                   Device->DevicePath,
 447                   &gEfiBlockIoProtocolGuid,
 448                   &Device->BlockIo,
 449                   &gEfiBlockIo2ProtocolGuid,
 450                   &Device->BlockIo2,
 451                   &gEfiDiskInfoProtocolGuid,
 452                   &Device->DiskInfo,
 453                   NULL
 454                   );
 455
 456   if (EFI_ERROR (Status)) {
 457     gBS->OpenProtocol (
 458            Controller,
 459            &gEfiNvmExpressPassThruProtocolGuid,
 460            (VOID **) &DummyInterface,
 461            This->DriverBindingHandle,
 462            Handle,
 463            EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER
 464            );
 465     return Status;
 466   }
 467
 468   //
 469   // If Storage Security Command Protocol is installed, then uninstall this protocol.
 470   //
 471   Status = gBS->OpenProtocol (
 472                   Handle,
 473                   &gEfiStorageSecurityCommandProtocolGuid,
 474                   (VOID **) &StorageSecurity,
 475                   This->DriverBindingHandle,
 476                   Controller,
 477                   EFI_OPEN_PROTOCOL_GET_PROTOCOL
 478                   );
 479
 480   if (!EFI_ERROR (Status)) {
 481     Status = gBS->UninstallProtocolInterface (
 482                     Handle,
 483                     &gEfiStorageSecurityCommandProtocolGuid,
 484                     &Device->StorageSecurity
 485                     );
 486     if (EFI_ERROR (Status)) {
 487       gBS->OpenProtocol (
 488         Controller,
 489         &gEfiNvmExpressPassThruProtocolGuid,
 490         (VOID **) &DummyInterface,
 491         This->DriverBindingHandle,
 492         Handle,
 493         EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER
 494         );
 495       return Status;
 496     }
 497   }
 498
 499   if(Device->DevicePath != NULL) {
 500     FreePool (Device->DevicePath);
 501   }
 502
 503   if (Device->ControllerNameTable != NULL) {
 504     FreeUnicodeStringTable (Device->ControllerNameTable);
 505   }
 506
 507   FreePool (Device);
 508
 509   return EFI_SUCCESS;
 510 }
 511
 512 /**
 513   Call back function when the timer event is signaled.
 514
 515   @param[in]  Event     The Event this notify function registered to.
 516   @param[in]  Context   Pointer to the context data registered to the
 517                         Event.
 518
 519 **/
 520 VOID
 521 EFIAPI
 522 ProcessAsyncTaskList (
 523   IN EFI_EVENT                    Event,
 524   IN VOID*                        Context
 525   )
 526 {
 527   NVME_CONTROLLER_PRIVATE_DATA         *Private;
 528   EFI_PCI_IO_PROTOCOL                  *PciIo;
 529   NVME_CQ                              *Cq;
 530   UINT16                               QueueId;
 531   UINT32                               Data;
 532   LIST_ENTRY                           *Link;
 533   LIST_ENTRY                           *NextLink;
 534   NVME_PASS_THRU_ASYNC_REQ             *AsyncRequest;
 535   NVME_BLKIO2_SUBTASK                  *Subtask;
 536   NVME_BLKIO2_REQUEST                  *BlkIo2Request;
 537   EFI_BLOCK_IO2_TOKEN                  *Token;
 538   BOOLEAN                              HasNewItem;
 539   EFI_STATUS                           Status;
 540
 541   Private    = (NVME_CONTROLLER_PRIVATE_DATA*)Context;
 542   QueueId    = 2;
 543   Cq         = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
 544   HasNewItem = FALSE;
 545   PciIo      = Private->PciIo;
 546
 547   //
 548   // Submit asynchronous subtasks to the NVMe Submission Queue
 549   //
 550   for (Link = GetFirstNode (&Private->UnsubmittedSubtasks);
 551        !IsNull (&Private->UnsubmittedSubtasks, Link);
 552        Link = NextLink) {
 553     NextLink      = GetNextNode (&Private->UnsubmittedSubtasks, Link);
 554     Subtask       = NVME_BLKIO2_SUBTASK_FROM_LINK (Link);
 555     BlkIo2Request = Subtask->BlockIo2Request;
 556     Token         = BlkIo2Request->Token;
 557     RemoveEntryList (Link);
 558     BlkIo2Request->UnsubmittedSubtaskNum--;
 559
 560     //
 561     // If any previous subtask fails, do not process subsequent ones.
 562     //
 563     if (Token->TransactionStatus != EFI_SUCCESS) {
 564       if (IsListEmpty (&BlkIo2Request->SubtasksQueue) &&
 565           BlkIo2Request->LastSubtaskSubmitted &&
 566           (BlkIo2Request->UnsubmittedSubtaskNum == 0)) {
 567         //
 568         // Remove the BlockIo2 request from the device asynchronous queue.
 569         //
 570         RemoveEntryList (&BlkIo2Request->Link);
 571         FreePool (BlkIo2Request);
 572         gBS->SignalEvent (Token->Event);
 573       }
 574
 575       FreePool (Subtask->CommandPacket->NvmeCmd);
 576       FreePool (Subtask->CommandPacket->NvmeCompletion);
 577       FreePool (Subtask->CommandPacket);
 578       FreePool (Subtask);
 579
 580       continue;
 581     }
 582
 583     Status = Private->Passthru.PassThru (
 584                                  &Private->Passthru,
 585                                  Subtask->NamespaceId,
 586                                  Subtask->CommandPacket,
 587                                  Subtask->Event
 588                                  );
 589     if (Status == EFI_NOT_READY) {
 590       InsertHeadList (&Private->UnsubmittedSubtasks, Link);
 591       BlkIo2Request->UnsubmittedSubtaskNum++;
 592       break;
 593     } else if (EFI_ERROR (Status)) {
 594       Token->TransactionStatus = EFI_DEVICE_ERROR;
 595
 596       if (IsListEmpty (&BlkIo2Request->SubtasksQueue) &&
 597           Subtask->IsLast) {
 598         //
 599         // Remove the BlockIo2 request from the device asynchronous queue.
 600         //
 601         RemoveEntryList (&BlkIo2Request->Link);
 602         FreePool (BlkIo2Request);
 603         gBS->SignalEvent (Token->Event);
 604       }
 605
 606       FreePool (Subtask->CommandPacket->NvmeCmd);
 607       FreePool (Subtask->CommandPacket->NvmeCompletion);
 608       FreePool (Subtask->CommandPacket);
 609       FreePool (Subtask);
 610     } else {
 611       InsertTailList (&BlkIo2Request->SubtasksQueue, Link);
 612       if (Subtask->IsLast) {
 613         BlkIo2Request->LastSubtaskSubmitted = TRUE;
 614       }
 615     }
 616   }
 617
 618   while (Cq->Pt != Private->Pt[QueueId]) {
 619     ASSERT (Cq->Sqid == QueueId);
 620
 621     HasNewItem = TRUE;
 622
 623     //
 624     // Find the command with given Command Id.
 625     //
 626     for (Link = GetFirstNode (&Private->AsyncPassThruQueue);
 627          !IsNull (&Private->AsyncPassThruQueue, Link);
 628          Link = NextLink) {
 629       NextLink = GetNextNode (&Private->AsyncPassThruQueue, Link);
 630       AsyncRequest = NVME_PASS_THRU_ASYNC_REQ_FROM_THIS (Link);
 631       if (AsyncRequest->CommandId == Cq->Cid) {
 632         //
 633         // Copy the Respose Queue entry for this command to the callers
 634         // response buffer.
 635         //
 636         CopyMem (
 637           AsyncRequest->Packet->NvmeCompletion,
 638           Cq,
 639           sizeof(EFI_NVM_EXPRESS_COMPLETION)
 640           );
 641
 642         //
 643         // Free the resources allocated before cmd submission
 644         //
 645         if (AsyncRequest->MapData != NULL) {
 646           PciIo->Unmap (PciIo, AsyncRequest->MapData);
 647         }
 648         if (AsyncRequest->MapMeta != NULL) {
 649           PciIo->Unmap (PciIo, AsyncRequest->MapMeta);
 650         }
 651         if (AsyncRequest->MapPrpList != NULL) {
 652           PciIo->Unmap (PciIo, AsyncRequest->MapPrpList);
 653         }
 654         if (AsyncRequest->PrpListHost != NULL) {
 655           PciIo->FreeBuffer (
 656                    PciIo,
 657                    AsyncRequest->PrpListNo,
 658                    AsyncRequest->PrpListHost
 659                    );
 660         }
 661
 662         RemoveEntryList (Link);
 663         gBS->SignalEvent (AsyncRequest->CallerEvent);
 664         FreePool (AsyncRequest);
 665
 666         //
 667         // Update submission queue head.
 668         //
 669         Private->AsyncSqHead = Cq->Sqhd;
 670         break;
 671       }
 672     }
 673
 674     Private->CqHdbl[QueueId].Cqh++;
 675     if (Private->CqHdbl[QueueId].Cqh > NVME_ASYNC_CCQ_SIZE) {
 676       Private->CqHdbl[QueueId].Cqh = 0;
 677       Private->Pt[QueueId] ^= 1;
 678     }
 679
 680     Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
 681   }
 682
 683   if (HasNewItem) {
 684     Data  = ReadUnaligned32 ((UINT32*)&Private->CqHdbl[QueueId]);
 685     PciIo->Mem.Write (
 686                  PciIo,
 687                  EfiPciIoWidthUint32,
 688                  NVME_BAR,
 689                  NVME_CQHDBL_OFFSET(QueueId, Private->Cap.Dstrd),
 690                  1,
 691                  &Data
 692                  );
 693   }
 694 }
 695
 696 /**
 697   Tests to see if this driver supports a given controller. If a child device is provided,
 698   it further tests to see if this driver supports creating a handle for the specified child device.
 699
 700   This function checks to see if the driver specified by This supports the device specified by
 701   ControllerHandle. Drivers will typically use the device path attached to
 702   ControllerHandle and/or the services from the bus I/O abstraction attached to
 703   ControllerHandle to determine if the driver supports ControllerHandle. This function
 704   may be called many times during platform initialization. In order to reduce boot times, the tests
 705   performed by this function must be very small, and take as little time as possible to execute. This
 706   function must not change the state of any hardware devices, and this function must be aware that the
 707   device specified by ControllerHandle may already be managed by the same driver or a
 708   different driver. This function must match its calls to AllocatePages() with FreePages(),
 709   AllocatePool() with FreePool(), and OpenProtocol() with CloseProtocol().
 710   Since ControllerHandle may have been previously started by the same driver, if a protocol is
 711   already in the opened state, then it must not be closed with CloseProtocol(). This is required
 712   to guarantee the state of ControllerHandle is not modified by this function.
 713
 714   @param[in]  This                 A pointer to the EFI_DRIVER_BINDING_PROTOCOL instance.
 715   @param[in]  ControllerHandle     The handle of the controller to test. This handle
 716                                    must support a protocol interface that supplies
 717                                    an I/O abstraction to the driver.
 718   @param[in]  RemainingDevicePath  A pointer to the remaining portion of a device path.  This
 719                                    parameter is ignored by device drivers, and is optional for bus
 720                                    drivers. For bus drivers, if this parameter is not NULL, then
 721                                    the bus driver must determine if the bus controller specified
 722                                    by ControllerHandle and the child controller specified
 723                                    by RemainingDevicePath are both supported by this
 724                                    bus driver.
 725
 726   @retval EFI_SUCCESS              The device specified by ControllerHandle and
 727                                    RemainingDevicePath is supported by the driver specified by This.
 728   @retval EFI_ALREADY_STARTED      The device specified by ControllerHandle and
 729                                    RemainingDevicePath is already being managed by the driver
 730                                    specified by This.
 731   @retval EFI_ACCESS_DENIED        The device specified by ControllerHandle and
 732                                    RemainingDevicePath is already being managed by a different
 733                                    driver or an application that requires exclusive access.
 734                                    Currently not implemented.
 735   @retval EFI_UNSUPPORTED          The device specified by ControllerHandle and
 736                                    RemainingDevicePath is not supported by the driver specified by This.
 737 **/
 738 EFI_STATUS
 739 EFIAPI
 740 NvmExpressDriverBindingSupported (
 741   IN EFI_DRIVER_BINDING_PROTOCOL  *This,
 742   IN EFI_HANDLE                   Controller,
 743   IN EFI_DEVICE_PATH_PROTOCOL     *RemainingDevicePath
 744   )
 745 {
 746   EFI_STATUS                Status;
 747   EFI_DEV_PATH_PTR          DevicePathNode;
 748   EFI_DEVICE_PATH_PROTOCOL  *ParentDevicePath;
 749   EFI_PCI_IO_PROTOCOL       *PciIo;
 750   UINT8                     ClassCode[3];
 751
 752   //
 753   // Check whether device path is valid
 754   //
 755   if (RemainingDevicePath != NULL) {
 756     //
 757     // Check if RemainingDevicePath is the End of Device Path Node,
 758     // if yes, go on checking other conditions
 759     //
 760     if (!IsDevicePathEnd (RemainingDevicePath)) {
 761       //
 762       // If RemainingDevicePath isn't the End of Device Path Node,
 763       // check its validation
 764       //
 765       DevicePathNode.DevPath = RemainingDevicePath;
 766
 767       if ((DevicePathNode.DevPath->Type    != MESSAGING_DEVICE_PATH) ||
 768           (DevicePathNode.DevPath->SubType != MSG_NVME_NAMESPACE_DP) ||
 769           (DevicePathNodeLength(DevicePathNode.DevPath) != sizeof(NVME_NAMESPACE_DEVICE_PATH))) {
 770          return EFI_UNSUPPORTED;
 771       }
 772     }
 773   }
 774
 775   //
 776   // Open the EFI Device Path protocol needed to perform the supported test
 777   //
 778   Status = gBS->OpenProtocol (
 779                   Controller,
 780                   &gEfiDevicePathProtocolGuid,
 781                   (VOID **) &ParentDevicePath,
 782                   This->DriverBindingHandle,
 783                   Controller,
 784                   EFI_OPEN_PROTOCOL_BY_DRIVER
 785                   );
 786   if (Status == EFI_ALREADY_STARTED) {
 787     return EFI_SUCCESS;
 788   }
 789
 790   if (EFI_ERROR (Status)) {
 791     return Status;
 792   }
 793
 794   //
 795   // Close protocol, don't use device path protocol in the Support() function
 796   //
 797   gBS->CloseProtocol (
 798          Controller,
 799          &gEfiDevicePathProtocolGuid,
 800          This->DriverBindingHandle,
 801          Controller
 802          );
 803
 804   //
 805   // Attempt to Open PCI I/O Protocol
 806   //
 807   Status = gBS->OpenProtocol (
 808                   Controller,
 809                   &gEfiPciIoProtocolGuid,
 810                   (VOID **) &PciIo,
 811                   This->DriverBindingHandle,
 812                   Controller,
 813                   EFI_OPEN_PROTOCOL_BY_DRIVER
 814                   );
 815   if (Status == EFI_ALREADY_STARTED) {
 816     return EFI_SUCCESS;
 817   }
 818
 819   if (EFI_ERROR (Status)) {
 820     return Status;
 821   }
 822
 823   //
 824   // Now further check the PCI header: Base class (offset 0x0B) and Sub Class (offset 0x0A).
 825   // This controller should be a Nvm Express controller.
 826   //
 827   Status = PciIo->Pci.Read (
 828                         PciIo,
 829                         EfiPciIoWidthUint8,
 830                         PCI_CLASSCODE_OFFSET,
 831                         sizeof (ClassCode),
 832                         ClassCode
 833                         );
 834   if (EFI_ERROR (Status)) {
 835     goto Done;
 836   }
 837
 838   //
 839   // Examine Nvm Express controller PCI Configuration table fields
 840   //
 841   if ((ClassCode[0] != PCI_IF_NVMHCI) || (ClassCode[1] != PCI_CLASS_MASS_STORAGE_NVM) || (ClassCode[2] != PCI_CLASS_MASS_STORAGE)) {
 842     Status = EFI_UNSUPPORTED;
 843   }
 844
 845 Done:
 846   gBS->CloseProtocol (
 847          Controller,
 848          &gEfiPciIoProtocolGuid,
 849          This->DriverBindingHandle,
 850          Controller
 851          );
 852
 853   return Status;
 854 }
 855
 856
 857 /**
 858   Starts a device controller or a bus controller.
 859
 860   The Start() function is designed to be invoked from the EFI boot service ConnectController().
 861   As a result, much of the error checking on the parameters to Start() has been moved into this
 862   common boot service. It is legal to call Start() from other locations,
 863   but the following calling restrictions must be followed or the system behavior will not be deterministic.
 864   1. ControllerHandle must be a valid EFI_HANDLE.
 865   2. If RemainingDevicePath is not NULL, then it must be a pointer to a naturally aligned
 866      EFI_DEVICE_PATH_PROTOCOL.
 867   3. Prior to calling Start(), the Supported() function for the driver specified by This must
 868      have been called with the same calling parameters, and Supported() must have returned EFI_SUCCESS.
 869
 870   @param[in]  This                 A pointer to the EFI_DRIVER_BINDING_PROTOCOL instance.
 871   @param[in]  ControllerHandle     The handle of the controller to start. This handle
 872                                    must support a protocol interface that supplies
 873                                    an I/O abstraction to the driver.
 874   @param[in]  RemainingDevicePath  A pointer to the remaining portion of a device path.  This
 875                                    parameter is ignored by device drivers, and is optional for bus
 876                                    drivers. For a bus driver, if this parameter is NULL, then handles
 877                                    for all the children of Controller are created by this driver.
 878                                    If this parameter is not NULL and the first Device Path Node is
 879                                    not the End of Device Path Node, then only the handle for the
 880                                    child device specified by the first Device Path Node of
 881                                    RemainingDevicePath is created by this driver.
 882                                    If the first Device Path Node of RemainingDevicePath is
 883                                    the End of Device Path Node, no child handle is created by this
 884                                    driver.
 885
 886   @retval EFI_SUCCESS              The device was started.
 887   @retval EFI_DEVICE_ERROR         The device could not be started due to a device error.Currently not implemented.
 888   @retval EFI_OUT_OF_RESOURCES     The request could not be completed due to a lack of resources.
 889   @retval Others                   The driver failded to start the device.
 890
 891 **/
 892 EFI_STATUS
 893 EFIAPI
 894 NvmExpressDriverBindingStart (
 895   IN EFI_DRIVER_BINDING_PROTOCOL  *This,
 896   IN EFI_HANDLE                   Controller,
 897   IN EFI_DEVICE_PATH_PROTOCOL     *RemainingDevicePath
 898   )
 899 {
 900   EFI_STATUS                          Status;
 901   EFI_PCI_IO_PROTOCOL                 *PciIo;
 902   NVME_CONTROLLER_PRIVATE_DATA        *Private;
 903   EFI_DEVICE_PATH_PROTOCOL            *ParentDevicePath;
 904   UINT32                              NamespaceId;
 905   EFI_PHYSICAL_ADDRESS                MappedAddr;
 906   UINTN                               Bytes;
 907   EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL  *Passthru;
 908
 909   DEBUG ((EFI_D_INFO, "NvmExpressDriverBindingStart: start\n"));
 910
 911   Private          = NULL;
 912   Passthru         = NULL;
 913   ParentDevicePath = NULL;
 914
 915   Status = gBS->OpenProtocol (
 916                   Controller,
 917                   &gEfiDevicePathProtocolGuid,
 918                   (VOID **) &ParentDevicePath,
 919                   This->DriverBindingHandle,
 920                   Controller,
 921                   EFI_OPEN_PROTOCOL_BY_DRIVER
 922                   );
 923   if ((EFI_ERROR (Status)) && (Status != EFI_ALREADY_STARTED)) {
 924     return Status;
 925   }
 926
 927   Status = gBS->OpenProtocol (
 928                   Controller,
 929                   &gEfiPciIoProtocolGuid,
 930                   (VOID **) &PciIo,
 931                   This->DriverBindingHandle,
 932                   Controller,
 933                   EFI_OPEN_PROTOCOL_BY_DRIVER
 934                   );
 935
 936   if (EFI_ERROR (Status) && (Status != EFI_ALREADY_STARTED)) {
 937     return Status;
 938   }
 939
 940   //
 941   // Check EFI_ALREADY_STARTED to reuse the original NVME_CONTROLLER_PRIVATE_DATA.
 942   //
 943   if (Status != EFI_ALREADY_STARTED) {
 944     Private = AllocateZeroPool (sizeof (NVME_CONTROLLER_PRIVATE_DATA));
 945
 946     if (Private == NULL) {
 947       DEBUG ((EFI_D_ERROR, "NvmExpressDriverBindingStart: allocating pool for Nvme Private Data failed!\n"));
 948       Status = EFI_OUT_OF_RESOURCES;
 949       goto Exit;
 950     }
 951
 952     //
 953     // 6 x 4kB aligned buffers will be carved out of this buffer.
 954     // 1st 4kB boundary is the start of the admin submission queue.
 955     // 2nd 4kB boundary is the start of the admin completion queue.
 956     // 3rd 4kB boundary is the start of I/O submission queue #1.
 957     // 4th 4kB boundary is the start of I/O completion queue #1.
 958     // 5th 4kB boundary is the start of I/O submission queue #2.
 959     // 6th 4kB boundary is the start of I/O completion queue #2.
 960     //
 961     // Allocate 6 pages of memory, then map it for bus master read and write.
 962     //
 963     Status = PciIo->AllocateBuffer (
 964                       PciIo,
 965                       AllocateAnyPages,
 966                       EfiBootServicesData,
 967                       6,
 968                       (VOID**)&Private->Buffer,
 969                       0
 970                       );
 971     if (EFI_ERROR (Status)) {
 972       goto Exit;
 973     }
 974
 975     Bytes = EFI_PAGES_TO_SIZE (6);
 976     Status = PciIo->Map (
 977                       PciIo,
 978                       EfiPciIoOperationBusMasterCommonBuffer,
 979                       Private->Buffer,
 980                       &Bytes,
 981                       &MappedAddr,
 982                       &Private->Mapping
 983                       );
 984
 985     if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (6))) {
 986       goto Exit;
 987     }
 988
 989     Private->BufferPciAddr = (UINT8 *)(UINTN)MappedAddr;
 990
 991     Private->Signature = NVME_CONTROLLER_PRIVATE_DATA_SIGNATURE;
 992     Private->ControllerHandle          = Controller;
 993     Private->ImageHandle               = This->DriverBindingHandle;
 994     Private->DriverBindingHandle       = This->DriverBindingHandle;
 995     Private->PciIo                     = PciIo;
 996     Private->ParentDevicePath          = ParentDevicePath;
 997     Private->Passthru.Mode             = &Private->PassThruMode;
 998     Private->Passthru.PassThru         = NvmExpressPassThru;
 999     Private->Passthru.GetNextNamespace = NvmExpressGetNextNamespace;
1000     Private->Passthru.BuildDevicePath  = NvmExpressBuildDevicePath;
1001     Private->Passthru.GetNamespace     = NvmExpressGetNamespace;
1002     CopyMem (&Private->PassThruMode, &gEfiNvmExpressPassThruMode, sizeof (EFI_NVM_EXPRESS_PASS_THRU_MODE));
1003     InitializeListHead (&Private->AsyncPassThruQueue);
1004     InitializeListHead (&Private->UnsubmittedSubtasks);
1005
1006     Status = NvmeControllerInit (Private);
1007     if (EFI_ERROR(Status)) {
1008       goto Exit;
1009     }
1010
1011     //
1012     // Start the asynchronous I/O completion monitor
1013     //
1014     Status = gBS->CreateEvent (
1015                     EVT_TIMER | EVT_NOTIFY_SIGNAL,
1016                     TPL_NOTIFY,
1017                     ProcessAsyncTaskList,
1018                     Private,
1019                     &Private->TimerEvent
1020                     );
1021     if (EFI_ERROR (Status)) {
1022       goto Exit;
1023     }
1024
1025     Status = gBS->SetTimer (
1026                     Private->TimerEvent,
1027                     TimerPeriodic,
1028                     NVME_HC_ASYNC_TIMER
1029                     );
1030     if (EFI_ERROR (Status)) {
1031       goto Exit;
1032     }
1033
1034     Status = gBS->InstallMultipleProtocolInterfaces (
1035                     &Controller,
1036                     &gEfiNvmExpressPassThruProtocolGuid,
1037                     &Private->Passthru,
1038                     NULL
1039                     );
1040     if (EFI_ERROR (Status)) {
1041       goto Exit;
1042     }
1043
1044     NvmeRegisterShutdownNotification ();
1045   } else {
1046     Status = gBS->OpenProtocol (
1047                     Controller,
1048                     &gEfiNvmExpressPassThruProtocolGuid,
1049                     (VOID **) &Passthru,
1050                     This->DriverBindingHandle,
1051                     Controller,
1052                     EFI_OPEN_PROTOCOL_GET_PROTOCOL
1053                     );
1054     if (EFI_ERROR (Status)) {
1055       goto Exit;
1056     }
1057
1058     Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (Passthru);
1059   }
1060
1061   if (RemainingDevicePath == NULL) {
1062     //
1063     // Enumerate all NVME namespaces in the controller
1064     //
1065     Status = DiscoverAllNamespaces (
1066                Private
1067                );
1068
1069   } else if (!IsDevicePathEnd (RemainingDevicePath)) {
1070     //
1071     // Enumerate the specified NVME namespace
1072     //
1073     Status = Private->Passthru.GetNamespace (
1074                                  &Private->Passthru,
1075                                  RemainingDevicePath,
1076                                  &NamespaceId
1077                                  );
1078
1079     if (!EFI_ERROR (Status)) {
1080       Status = EnumerateNvmeDevNamespace (
1081                  Private,
1082                  NamespaceId
1083                  );
1084     }
1085   }
1086
1087   DEBUG ((EFI_D_INFO, "NvmExpressDriverBindingStart: end successfully\n"));
1088   return EFI_SUCCESS;
1089
1090 Exit:
1091   if ((Private != NULL) && (Private->Mapping != NULL)) {
1092     PciIo->Unmap (PciIo, Private->Mapping);
1093   }
1094
1095   if ((Private != NULL) && (Private->Buffer != NULL)) {
1096     PciIo->FreeBuffer (PciIo, 6, Private->Buffer);
1097   }
1098
1099   if ((Private != NULL) && (Private->ControllerData != NULL)) {
1100     FreePool (Private->ControllerData);
1101   }
1102
1103   if (Private != NULL) {
1104     if (Private->TimerEvent != NULL) {
1105       gBS->CloseEvent (Private->TimerEvent);
1106     }
1107
1108     FreePool (Private);
1109   }
1110
1111   gBS->CloseProtocol (
1112          Controller,
1113          &gEfiPciIoProtocolGuid,
1114          This->DriverBindingHandle,
1115          Controller
1116          );
1117
1118   gBS->CloseProtocol (
1119          Controller,
1120          &gEfiDevicePathProtocolGuid,
1121          This->DriverBindingHandle,
1122          Controller
1123          );
1124
1125   DEBUG ((EFI_D_INFO, "NvmExpressDriverBindingStart: end with %r\n", Status));
1126
1127   return Status;
1128 }
1129
1130
1131 /**
1132   Stops a device controller or a bus controller.
1133
1134   The Stop() function is designed to be invoked from the EFI boot service DisconnectController().
1135   As a result, much of the error checking on the parameters to Stop() has been moved
1136   into this common boot service. It is legal to call Stop() from other locations,
1137   but the following calling restrictions must be followed or the system behavior will not be deterministic.
1138   1. ControllerHandle must be a valid EFI_HANDLE that was used on a previous call to this
1139      same driver's Start() function.
1140   2. The first NumberOfChildren handles of ChildHandleBuffer must all be a valid
1141      EFI_HANDLE. In addition, all of these handles must have been created in this driver's
1142      Start() function, and the Start() function must have called OpenProtocol() on
1143      ControllerHandle with an Attribute of EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER.
1144
1145   @param[in]  This              A pointer to the EFI_DRIVER_BINDING_PROTOCOL instance.
1146   @param[in]  ControllerHandle  A handle to the device being stopped. The handle must
1147                                 support a bus specific I/O protocol for the driver
1148                                 to use to stop the device.
1149   @param[in]  NumberOfChildren  The number of child device handles in ChildHandleBuffer.
1150   @param[in]  ChildHandleBuffer An array of child handles to be freed. May be NULL
1151                                 if NumberOfChildren is 0.
1152
1153   @retval EFI_SUCCESS           The device was stopped.
1154   @retval EFI_DEVICE_ERROR      The device could not be stopped due to a device error.
1155
1156 **/
1157 EFI_STATUS
1158 EFIAPI
1159 NvmExpressDriverBindingStop (
1160   IN  EFI_DRIVER_BINDING_PROTOCOL     *This,
1161   IN  EFI_HANDLE                      Controller,
1162   IN  UINTN                           NumberOfChildren,
1163   IN  EFI_HANDLE                      *ChildHandleBuffer
1164   )
1165 {
1166   EFI_STATUS                          Status;
1167   BOOLEAN                             AllChildrenStopped;
1168   UINTN                               Index;
1169   NVME_CONTROLLER_PRIVATE_DATA        *Private;
1170   EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL  *PassThru;
1171   BOOLEAN                             IsEmpty;
1172   EFI_TPL                             OldTpl;
1173
1174   if (NumberOfChildren == 0) {
1175     Status = gBS->OpenProtocol (
1176                     Controller,
1177                     &gEfiNvmExpressPassThruProtocolGuid,
1178                     (VOID **) &PassThru,
1179                     This->DriverBindingHandle,
1180                     Controller,
1181                     EFI_OPEN_PROTOCOL_GET_PROTOCOL
1182                     );
1183
1184     if (!EFI_ERROR (Status)) {
1185       Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (PassThru);
1186
1187       //
1188       // Wait for the asynchronous PassThru queue to become empty.
1189       //
1190       while (TRUE) {
1191         OldTpl  = gBS->RaiseTPL (TPL_NOTIFY);
1192         IsEmpty = IsListEmpty (&Private->AsyncPassThruQueue) &&
1193                   IsListEmpty (&Private->UnsubmittedSubtasks);
1194         gBS->RestoreTPL (OldTpl);
1195
1196         if (IsEmpty) {
1197           break;
1198         }
1199
1200         gBS->Stall (100);
1201       }
1202
1203       gBS->UninstallMultipleProtocolInterfaces (
1204             Controller,
1205             &gEfiNvmExpressPassThruProtocolGuid,
1206             PassThru,
1207             NULL
1208             );
1209
1210       if (Private->TimerEvent != NULL) {
1211         gBS->CloseEvent (Private->TimerEvent);
1212       }
1213
1214       if (Private->Mapping != NULL) {
1215         Private->PciIo->Unmap (Private->PciIo, Private->Mapping);
1216       }
1217
1218       if (Private->Buffer != NULL) {
1219         Private->PciIo->FreeBuffer (Private->PciIo, 6, Private->Buffer);
1220       }
1221
1222       FreePool (Private->ControllerData);
1223       FreePool (Private);
1224     }
1225
1226     gBS->CloseProtocol (
1227           Controller,
1228           &gEfiPciIoProtocolGuid,
1229           This->DriverBindingHandle,
1230           Controller
1231           );
1232     gBS->CloseProtocol (
1233           Controller,
1234           &gEfiDevicePathProtocolGuid,
1235           This->DriverBindingHandle,
1236           Controller
1237           );
1238
1239     NvmeUnregisterShutdownNotification ();
1240
1241     return EFI_SUCCESS;
1242   }
1243
1244   AllChildrenStopped = TRUE;
1245
1246   for (Index = 0; Index < NumberOfChildren; Index++) {
1247     Status = UnregisterNvmeNamespace (This, Controller, ChildHandleBuffer[Index]);
1248     if (EFI_ERROR (Status)) {
1249       AllChildrenStopped = FALSE;
1250     }
1251   }
1252
1253   if (!AllChildrenStopped) {
1254     return EFI_DEVICE_ERROR;
1255   }
1256
1257   return EFI_SUCCESS;
1258 }
1259
1260 /**
1261   This is the unload handle for the NVM Express driver.
1262
1263   Disconnect the driver specified by ImageHandle from the NVMe device in the handle database.
1264   Uninstall all the protocols installed in the driver.
1265
1266   @param[in]  ImageHandle       The drivers' driver image.
1267
1268   @retval EFI_SUCCESS           The image is unloaded.
1269   @retval Others                Failed to unload the image.
1270
1271 **/
1272 EFI_STATUS
1273 EFIAPI
1274 NvmExpressUnload (
1275   IN EFI_HANDLE             ImageHandle
1276   )
1277 {
1278   EFI_STATUS                        Status;
1279   EFI_HANDLE                        *DeviceHandleBuffer;
1280   UINTN                             DeviceHandleCount;
1281   UINTN                             Index;
1282   EFI_COMPONENT_NAME_PROTOCOL       *ComponentName;
1283   EFI_COMPONENT_NAME2_PROTOCOL      *ComponentName2;
1284
1285   //
1286   // Get the list of the device handles managed by this driver.
1287   // If there is an error getting the list, then means the driver
1288   // doesn't manage any device. At this way, we would only close
1289   // those protocols installed at image handle.
1290   //
1291   DeviceHandleBuffer = NULL;
1292   Status = gBS->LocateHandleBuffer (
1293                   ByProtocol,
1294                   &gEfiNvmExpressPassThruProtocolGuid,
1295                   NULL,
1296                   &DeviceHandleCount,
1297                   &DeviceHandleBuffer
1298                   );
1299
1300   if (!EFI_ERROR (Status)) {
1301     //
1302     // Disconnect the driver specified by ImageHandle from all
1303     // the devices in the handle database.
1304     //
1305     for (Index = 0; Index < DeviceHandleCount; Index++) {
1306       Status = gBS->DisconnectController (
1307                       DeviceHandleBuffer[Index],
1308                       ImageHandle,
1309                       NULL
1310                       );
1311       if (EFI_ERROR (Status)) {
1312         goto EXIT;
1313       }
1314     }
1315   }
1316
1317   //
1318   // Uninstall all the protocols installed in the driver entry point
1319   //
1320   Status = gBS->UninstallMultipleProtocolInterfaces (
1321                   ImageHandle,
1322                   &gEfiDriverBindingProtocolGuid,
1323                   &gNvmExpressDriverBinding,
1324                   &gEfiDriverSupportedEfiVersionProtocolGuid,
1325                   &gNvmExpressDriverSupportedEfiVersion,
1326                   NULL
1327                   );
1328
1329   if (EFI_ERROR (Status)) {
1330     goto EXIT;
1331   }
1332
1333   //
1334   // Note we have to one by one uninstall the following protocols.
1335   // It's because some of them are optionally installed based on
1336   // the following PCD settings.
1337   //   gEfiMdePkgTokenSpaceGuid.PcdDriverDiagnosticsDisable
1338   //   gEfiMdePkgTokenSpaceGuid.PcdComponentNameDisable
1339   //   gEfiMdePkgTokenSpaceGuid.PcdDriverDiagnostics2Disable
1340   //   gEfiMdePkgTokenSpaceGuid.PcdComponentName2Disable
1341   //
1342   Status = gBS->HandleProtocol (
1343                   ImageHandle,
1344                   &gEfiComponentNameProtocolGuid,
1345                   (VOID **) &ComponentName
1346                   );
1347   if (!EFI_ERROR (Status)) {
1348     gBS->UninstallProtocolInterface (
1349            ImageHandle,
1350            &gEfiComponentNameProtocolGuid,
1351            ComponentName
1352            );
1353   }
1354
1355   Status = gBS->HandleProtocol (
1356                   ImageHandle,
1357                   &gEfiComponentName2ProtocolGuid,
1358                   (VOID **) &ComponentName2
1359                   );
1360   if (!EFI_ERROR (Status)) {
1361     gBS->UninstallProtocolInterface (
1362            ImageHandle,
1363            &gEfiComponentName2ProtocolGuid,
1364            ComponentName2
1365            );
1366   }
1367
1368   Status = EFI_SUCCESS;
1369
1370 EXIT:
1371   //
1372   // Free the buffer containing the list of handles from the handle database
1373   //
1374   if (DeviceHandleBuffer != NULL) {
1375     gBS->FreePool (DeviceHandleBuffer);
1376   }
1377   return Status;
1378 }
1379
1380 /**
1381   The entry point for Nvm Express driver, used to install Nvm Express driver on the ImageHandle.
1382
1383   @param  ImageHandle   The firmware allocated handle for this driver image.
1384   @param  SystemTable   Pointer to the EFI system table.
1385
1386   @retval EFI_SUCCESS   Driver loaded.
1387   @retval other         Driver not loaded.
1388
1389 **/
1390 EFI_STATUS
1391 EFIAPI
1392 NvmExpressDriverEntry (
1393   IN EFI_HANDLE        ImageHandle,
1394   IN EFI_SYSTEM_TABLE  *SystemTable
1395   )
1396 {
1397   EFI_STATUS              Status;
1398
1399   Status = EfiLibInstallDriverBindingComponentName2 (
1400              ImageHandle,
1401              SystemTable,
1402              &gNvmExpressDriverBinding,
1403              ImageHandle,
1404              &gNvmExpressComponentName,
1405              &gNvmExpressComponentName2
1406              );
1407   ASSERT_EFI_ERROR (Status);
1408
1409   //
1410   // Install EFI Driver Supported EFI Version Protocol required for
1411   // EFI drivers that are on PCI and other plug in cards.
1412   //
1413   gNvmExpressDriverSupportedEfiVersion.FirmwareVersion = 0x00020028;
1414   Status = gBS->InstallMultipleProtocolInterfaces (
1415                   &ImageHandle,
1416                   &gEfiDriverSupportedEfiVersionProtocolGuid,
1417                   &gNvmExpressDriverSupportedEfiVersion,
1418                   NULL
1419                   );
1420   ASSERT_EFI_ERROR (Status);
1421   return Status;
1422 }