X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=MdeModulePkg%2FBus%2FPci%2FNvmExpressDxe%2FNvmExpress.c;h=dea14f1a446cc6aee8953dca83e29b65a4c52678;hb=79f2734e5a7bc2e5256eb0e599f45407855159c7;hp=321dbdef9756d641e8f16c87f1dd0efe47109e17;hpb=6523ae8baa91aea55ebcfb0d4ac056eb1962b723;p=mirror_edk2.git
diff --git a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.c b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.c
index 321dbdef97..dea14f1a44 100644
--- a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.c
+++ b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.c
@@ -2,14 +2,8 @@
NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows
NVM Express specification.
- Copyright (c) 2013 - 2016, Intel Corporation. All rights reserved.
- This program and the accompanying materials
- are licensed and made available under the terms and conditions of the BSD License
- which accompanies this distribution. The full text of the license may be found at
- http://opensource.org/licenses/bsd-license.php.
-
- THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
- WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
+ Copyright (c) 2013 - 2017, Intel Corporation. All rights reserved.
+ SPDX-License-Identifier: BSD-2-Clause-Patent
**/
@@ -18,7 +12,7 @@
//
// NVM Express Driver Binding Protocol Instance
//
-EFI_DRIVER_BINDING_PROTOCOL gNvmExpressDriverBinding = {
+EFI_DRIVER_BINDING_PROTOCOL gNvmExpressDriverBinding = {
NvmExpressDriverBindingSupported,
NvmExpressDriverBindingStart,
NvmExpressDriverBindingStop,
@@ -30,7 +24,7 @@ EFI_DRIVER_BINDING_PROTOCOL gNvmExpressDriverBinding = {
//
// NVM Express EFI Driver Supported EFI Version Protocol Instance
//
-EFI_DRIVER_SUPPORTED_EFI_VERSION_PROTOCOL gNvmExpressDriverSupportedEfiVersion = {
+EFI_DRIVER_SUPPORTED_EFI_VERSION_PROTOCOL gNvmExpressDriverSupportedEfiVersion = {
sizeof (EFI_DRIVER_SUPPORTED_EFI_VERSION_PROTOCOL), // Size of Protocol structure.
0 // Version number to be filled at start up.
};
@@ -38,8 +32,11 @@ EFI_DRIVER_SUPPORTED_EFI_VERSION_PROTOCOL gNvmExpressDriverSupportedEfiVersion =
//
// Template for NVM Express Pass Thru Mode data structure.
//
-GLOBAL_REMOVE_IF_UNREFERENCED EFI_NVM_EXPRESS_PASS_THRU_MODE gEfiNvmExpressPassThruMode = {
- EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL | EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL | EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_CMD_SET_NVM,
+GLOBAL_REMOVE_IF_UNREFERENCED EFI_NVM_EXPRESS_PASS_THRU_MODE gEfiNvmExpressPassThruMode = {
+ EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL |
+ EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL |
+ EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_NONBLOCKIO |
+ EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_CMD_SET_NVM,
sizeof (UINTN),
0x10100
};
@@ -59,23 +56,24 @@ GLOBAL_REMOVE_IF_UNREFERENCED EFI_NVM_EXPRESS_PASS_THRU_MODE gEfiNvmExpressPassT
**/
EFI_STATUS
EnumerateNvmeDevNamespace (
- IN NVME_CONTROLLER_PRIVATE_DATA *Private,
- UINT32 NamespaceId
+ IN NVME_CONTROLLER_PRIVATE_DATA *Private,
+ UINT32 NamespaceId
)
{
- NVME_ADMIN_NAMESPACE_DATA *NamespaceData;
- EFI_DEVICE_PATH_PROTOCOL *NewDevicePathNode;
- EFI_DEVICE_PATH_PROTOCOL *DevicePath;
- EFI_HANDLE DeviceHandle;
- EFI_DEVICE_PATH_PROTOCOL *ParentDevicePath;
- EFI_DEVICE_PATH_PROTOCOL *RemainingDevicePath;
- NVME_DEVICE_PRIVATE_DATA *Device;
- EFI_STATUS Status;
- UINT32 Lbads;
- UINT32 Flbas;
- UINT32 LbaFmtIdx;
- UINT8 Sn[21];
- UINT8 Mn[41];
+ NVME_ADMIN_NAMESPACE_DATA *NamespaceData;
+ EFI_DEVICE_PATH_PROTOCOL *NewDevicePathNode;
+ EFI_DEVICE_PATH_PROTOCOL *DevicePath;
+ EFI_HANDLE DeviceHandle;
+ EFI_DEVICE_PATH_PROTOCOL *ParentDevicePath;
+ EFI_DEVICE_PATH_PROTOCOL *RemainingDevicePath;
+ NVME_DEVICE_PRIVATE_DATA *Device;
+ EFI_STATUS Status;
+ UINT32 Lbads;
+ UINT32 Flbas;
+ UINT32 LbaFmtIdx;
+ UINT8 Sn[21];
+ UINT8 Mn[41];
+ VOID *DummyInterface;
NewDevicePathNode = NULL;
DevicePath = NULL;
@@ -84,8 +82,8 @@ EnumerateNvmeDevNamespace (
//
// Allocate a buffer for Identify Namespace data
//
- NamespaceData = AllocateZeroPool(sizeof (NVME_ADMIN_NAMESPACE_DATA));
- if(NamespaceData == NULL) {
+ NamespaceData = AllocateZeroPool (sizeof (NVME_ADMIN_NAMESPACE_DATA));
+ if (NamespaceData == NULL) {
return EFI_OUT_OF_RESOURCES;
}
@@ -98,9 +96,10 @@ EnumerateNvmeDevNamespace (
NamespaceId,
(VOID *)NamespaceData
);
- if (EFI_ERROR(Status)) {
+ if (EFI_ERROR (Status)) {
goto Exit;
}
+
//
// Validate Namespace
//
@@ -110,7 +109,7 @@ EnumerateNvmeDevNamespace (
//
// allocate device private data for each discovered namespace
//
- Device = AllocateZeroPool(sizeof(NVME_DEVICE_PRIVATE_DATA));
+ Device = AllocateZeroPool (sizeof (NVME_DEVICE_PRIVATE_DATA));
if (Device == NULL) {
Status = EFI_OUT_OF_RESOURCES;
goto Exit;
@@ -119,9 +118,9 @@ EnumerateNvmeDevNamespace (
//
// Initialize SSD namespace instance data
//
- Device->Signature = NVME_DEVICE_PRIVATE_DATA_SIGNATURE;
- Device->NamespaceId = NamespaceId;
- Device->NamespaceUuid = NamespaceData->Eui64;
+ Device->Signature = NVME_DEVICE_PRIVATE_DATA_SIGNATURE;
+ Device->NamespaceId = NamespaceId;
+ Device->NamespaceUuid = NamespaceData->Eui64;
Device->ControllerHandle = Private->ControllerHandle;
Device->DriverBindingHandle = Private->DriverBindingHandle;
@@ -130,16 +129,32 @@ EnumerateNvmeDevNamespace (
//
// Build BlockIo media structure
//
- Device->Media.MediaId = 0;
- Device->Media.RemovableMedia = FALSE;
- Device->Media.MediaPresent = TRUE;
+ Device->Media.MediaId = 0;
+ Device->Media.RemovableMedia = FALSE;
+ Device->Media.MediaPresent = TRUE;
Device->Media.LogicalPartition = FALSE;
- Device->Media.ReadOnly = FALSE;
- Device->Media.WriteCaching = FALSE;
+ Device->Media.ReadOnly = FALSE;
+ Device->Media.WriteCaching = FALSE;
+ Device->Media.IoAlign = Private->PassThruMode.IoAlign;
Flbas = NamespaceData->Flbas;
LbaFmtIdx = Flbas & 0xF;
- Lbads = NamespaceData->LbaFormat[LbaFmtIdx].Lbads;
+
+ //
+ // Currently this NVME driver only suport Metadata Size == 0
+ //
+ if (NamespaceData->LbaFormat[LbaFmtIdx].Ms != 0) {
+ DEBUG ((
+ DEBUG_ERROR,
+ "NVME IDENTIFY NAMESPACE [%d] Ms(%d) is not supported.\n",
+ NamespaceId,
+ NamespaceData->LbaFormat[LbaFmtIdx].Ms
+ ));
+ Status = EFI_UNSUPPORTED;
+ goto Exit;
+ }
+
+ Lbads = NamespaceData->LbaFormat[LbaFmtIdx].Lbads;
Device->Media.BlockSize = (UINT32)1 << Lbads;
Device->Media.LastBlock = NamespaceData->Nsze - 1;
@@ -149,12 +164,22 @@ EnumerateNvmeDevNamespace (
//
// Create BlockIo Protocol instance
//
- Device->BlockIo.Revision = EFI_BLOCK_IO_PROTOCOL_REVISION2;
- Device->BlockIo.Media = &Device->Media;
- Device->BlockIo.Reset = NvmeBlockIoReset;
- Device->BlockIo.ReadBlocks = NvmeBlockIoReadBlocks;
- Device->BlockIo.WriteBlocks = NvmeBlockIoWriteBlocks;
- Device->BlockIo.FlushBlocks = NvmeBlockIoFlushBlocks;
+ Device->BlockIo.Revision = EFI_BLOCK_IO_PROTOCOL_REVISION2;
+ Device->BlockIo.Media = &Device->Media;
+ Device->BlockIo.Reset = NvmeBlockIoReset;
+ Device->BlockIo.ReadBlocks = NvmeBlockIoReadBlocks;
+ Device->BlockIo.WriteBlocks = NvmeBlockIoWriteBlocks;
+ Device->BlockIo.FlushBlocks = NvmeBlockIoFlushBlocks;
+
+ //
+ // Create BlockIo2 Protocol instance
+ //
+ Device->BlockIo2.Media = &Device->Media;
+ Device->BlockIo2.Reset = NvmeBlockIoResetEx;
+ Device->BlockIo2.ReadBlocksEx = NvmeBlockIoReadBlocksEx;
+ Device->BlockIo2.WriteBlocksEx = NvmeBlockIoWriteBlocksEx;
+ Device->BlockIo2.FlushBlocksEx = NvmeBlockIoFlushBlocksEx;
+ InitializeListHead (&Device->AsyncQueue);
//
// Create StorageSecurityProtocol Instance
@@ -177,7 +202,7 @@ EnumerateNvmeDevNamespace (
&NewDevicePathNode
);
- if (EFI_ERROR(Status)) {
+ if (EFI_ERROR (Status)) {
goto Exit;
}
@@ -190,10 +215,10 @@ EnumerateNvmeDevNamespace (
goto Exit;
}
- DeviceHandle = NULL;
+ DeviceHandle = NULL;
RemainingDevicePath = DevicePath;
- Status = gBS->LocateDevicePath (&gEfiDevicePathProtocolGuid, &RemainingDevicePath, &DeviceHandle);
- if (!EFI_ERROR (Status) && (DeviceHandle != NULL) && IsDevicePathEnd(RemainingDevicePath)) {
+ Status = gBS->LocateDevicePath (&gEfiDevicePathProtocolGuid, &RemainingDevicePath, &DeviceHandle);
+ if (!EFI_ERROR (Status) && (DeviceHandle != NULL) && IsDevicePathEnd (RemainingDevicePath)) {
Status = EFI_ALREADY_STARTED;
FreePool (DevicePath);
goto Exit;
@@ -212,12 +237,14 @@ EnumerateNvmeDevNamespace (
Device->DevicePath,
&gEfiBlockIoProtocolGuid,
&Device->BlockIo,
+ &gEfiBlockIo2ProtocolGuid,
+ &Device->BlockIo2,
&gEfiDiskInfoProtocolGuid,
&Device->DiskInfo,
NULL
);
- if(EFI_ERROR(Status)) {
+ if (EFI_ERROR (Status)) {
goto Exit;
}
@@ -231,13 +258,15 @@ EnumerateNvmeDevNamespace (
EFI_NATIVE_INTERFACE,
&Device->StorageSecurity
);
- if(EFI_ERROR(Status)) {
+ if (EFI_ERROR (Status)) {
gBS->UninstallMultipleProtocolInterfaces (
- &Device->DeviceHandle,
+ Device->DeviceHandle,
&gEfiDevicePathProtocolGuid,
Device->DevicePath,
&gEfiBlockIoProtocolGuid,
&Device->BlockIo,
+ &gEfiBlockIo2ProtocolGuid,
+ &Device->BlockIo2,
&gEfiDiskInfoProtocolGuid,
&Device->DiskInfo,
NULL
@@ -249,7 +278,7 @@ EnumerateNvmeDevNamespace (
gBS->OpenProtocol (
Private->ControllerHandle,
&gEfiNvmExpressPassThruProtocolGuid,
- (VOID **) &Private->Passthru,
+ (VOID **)&DummyInterface,
Private->DriverBindingHandle,
Device->DeviceHandle,
EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER
@@ -258,11 +287,11 @@ EnumerateNvmeDevNamespace (
//
// Dump NvmExpress Identify Namespace Data
//
- DEBUG ((EFI_D_INFO, " == NVME IDENTIFY NAMESPACE [%d] DATA ==\n", NamespaceId));
- DEBUG ((EFI_D_INFO, " NSZE : 0x%x\n", NamespaceData->Nsze));
- DEBUG ((EFI_D_INFO, " NCAP : 0x%x\n", NamespaceData->Ncap));
- DEBUG ((EFI_D_INFO, " NUSE : 0x%x\n", NamespaceData->Nuse));
- DEBUG ((EFI_D_INFO, " LBAF0.LBADS : 0x%x\n", (NamespaceData->LbaFormat[0].Lbads)));
+ DEBUG ((DEBUG_INFO, " == NVME IDENTIFY NAMESPACE [%d] DATA ==\n", NamespaceId));
+ DEBUG ((DEBUG_INFO, " NSZE : 0x%x\n", NamespaceData->Nsze));
+ DEBUG ((DEBUG_INFO, " NCAP : 0x%x\n", NamespaceData->Ncap));
+ DEBUG ((DEBUG_INFO, " NUSE : 0x%x\n", NamespaceData->Nuse));
+ DEBUG ((DEBUG_INFO, " LBAF0.LBADS : 0x%x\n", (NamespaceData->LbaFormat[0].Lbads)));
//
// Build controller name for Component Name (2) protocol.
@@ -291,7 +320,7 @@ EnumerateNvmeDevNamespace (
}
Exit:
- if(NamespaceData != NULL) {
+ if (NamespaceData != NULL) {
FreePool (NamespaceData);
}
@@ -299,12 +328,14 @@ Exit:
FreePool (NewDevicePathNode);
}
- if(EFI_ERROR(Status) && (Device != NULL) && (Device->DevicePath != NULL)) {
+ if (EFI_ERROR (Status) && (Device != NULL) && (Device->DevicePath != NULL)) {
FreePool (Device->DevicePath);
}
- if(EFI_ERROR(Status) && (Device != NULL)) {
+
+ if (EFI_ERROR (Status) && (Device != NULL)) {
FreePool (Device);
}
+
return Status;
}
@@ -320,15 +351,15 @@ Exit:
**/
EFI_STATUS
DiscoverAllNamespaces (
- IN NVME_CONTROLLER_PRIVATE_DATA *Private
+ IN NVME_CONTROLLER_PRIVATE_DATA *Private
)
{
- EFI_STATUS Status;
- UINT32 NamespaceId;
- EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *Passthru;
+ EFI_STATUS Status;
+ UINT32 NamespaceId;
+ EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *Passthru;
- NamespaceId = 0xFFFFFFFF;
- Passthru = &Private->Passthru;
+ NamespaceId = 0xFFFFFFFF;
+ Passthru = &Private->Passthru;
while (TRUE) {
Status = Passthru->GetNextNamespace (
@@ -345,7 +376,7 @@ DiscoverAllNamespaces (
NamespaceId
);
- if (EFI_ERROR(Status)) {
+ if (EFI_ERROR (Status)) {
continue;
}
}
@@ -369,23 +400,25 @@ DiscoverAllNamespaces (
**/
EFI_STATUS
UnregisterNvmeNamespace (
- IN EFI_DRIVER_BINDING_PROTOCOL *This,
- IN EFI_HANDLE Controller,
- IN EFI_HANDLE Handle
+ IN EFI_DRIVER_BINDING_PROTOCOL *This,
+ IN EFI_HANDLE Controller,
+ IN EFI_HANDLE Handle
)
{
- EFI_STATUS Status;
- EFI_BLOCK_IO_PROTOCOL *BlockIo;
- NVME_DEVICE_PRIVATE_DATA *Device;
- NVME_CONTROLLER_PRIVATE_DATA *Private;
- EFI_STORAGE_SECURITY_COMMAND_PROTOCOL *StorageSecurity;
+ EFI_STATUS Status;
+ EFI_BLOCK_IO_PROTOCOL *BlockIo;
+ NVME_DEVICE_PRIVATE_DATA *Device;
+ EFI_STORAGE_SECURITY_COMMAND_PROTOCOL *StorageSecurity;
+ BOOLEAN IsEmpty;
+ EFI_TPL OldTpl;
+ VOID *DummyInterface;
BlockIo = NULL;
Status = gBS->OpenProtocol (
Handle,
&gEfiBlockIoProtocolGuid,
- (VOID **) &BlockIo,
+ (VOID **)&BlockIo,
This->DriverBindingHandle,
Controller,
EFI_OPEN_PROTOCOL_GET_PROTOCOL
@@ -394,8 +427,22 @@ UnregisterNvmeNamespace (
return Status;
}
- Device = NVME_DEVICE_PRIVATE_DATA_FROM_BLOCK_IO (BlockIo);
- Private = Device->Controller;
+ Device = NVME_DEVICE_PRIVATE_DATA_FROM_BLOCK_IO (BlockIo);
+
+ //
+ // Wait for the device's asynchronous I/O queue to become empty.
+ //
+ while (TRUE) {
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);
+ IsEmpty = IsListEmpty (&Device->AsyncQueue);
+ gBS->RestoreTPL (OldTpl);
+
+ if (IsEmpty) {
+ break;
+ }
+
+ gBS->Stall (100);
+ }
//
// Close the child handle
@@ -417,6 +464,8 @@ UnregisterNvmeNamespace (
Device->DevicePath,
&gEfiBlockIoProtocolGuid,
&Device->BlockIo,
+ &gEfiBlockIo2ProtocolGuid,
+ &Device->BlockIo2,
&gEfiDiskInfoProtocolGuid,
&Device->DiskInfo,
NULL
@@ -426,7 +475,7 @@ UnregisterNvmeNamespace (
gBS->OpenProtocol (
Controller,
&gEfiNvmExpressPassThruProtocolGuid,
- (VOID **) &Private->Passthru,
+ (VOID **)&DummyInterface,
This->DriverBindingHandle,
Handle,
EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER
@@ -440,7 +489,7 @@ UnregisterNvmeNamespace (
Status = gBS->OpenProtocol (
Handle,
&gEfiStorageSecurityCommandProtocolGuid,
- (VOID **) &StorageSecurity,
+ (VOID **)&StorageSecurity,
This->DriverBindingHandle,
Controller,
EFI_OPEN_PROTOCOL_GET_PROTOCOL
@@ -454,18 +503,18 @@ UnregisterNvmeNamespace (
);
if (EFI_ERROR (Status)) {
gBS->OpenProtocol (
- Controller,
- &gEfiNvmExpressPassThruProtocolGuid,
- (VOID **) &Private->Passthru,
- This->DriverBindingHandle,
- Handle,
- EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER
- );
+ Controller,
+ &gEfiNvmExpressPassThruProtocolGuid,
+ (VOID **)&DummyInterface,
+ This->DriverBindingHandle,
+ Handle,
+ EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER
+ );
return Status;
}
}
- if(Device->DevicePath != NULL) {
+ if (Device->DevicePath != NULL) {
FreePool (Device->DevicePath);
}
@@ -478,6 +527,197 @@ UnregisterNvmeNamespace (
return EFI_SUCCESS;
}
+/**
+ Call back function when the timer event is signaled.
+
+ @param[in] Event The Event this notify function registered to.
+ @param[in] Context Pointer to the context data registered to the
+ Event.
+
+**/
+VOID
+EFIAPI
+ProcessAsyncTaskList (
+ IN EFI_EVENT Event,
+ IN VOID *Context
+ )
+{
+ NVME_CONTROLLER_PRIVATE_DATA *Private;
+ EFI_PCI_IO_PROTOCOL *PciIo;
+ NVME_CQ *Cq;
+ UINT16 QueueId;
+ UINT32 Data;
+ LIST_ENTRY *Link;
+ LIST_ENTRY *NextLink;
+ NVME_PASS_THRU_ASYNC_REQ *AsyncRequest;
+ NVME_BLKIO2_SUBTASK *Subtask;
+ NVME_BLKIO2_REQUEST *BlkIo2Request;
+ EFI_BLOCK_IO2_TOKEN *Token;
+ BOOLEAN HasNewItem;
+ EFI_STATUS Status;
+
+ Private = (NVME_CONTROLLER_PRIVATE_DATA *)Context;
+ QueueId = 2;
+ Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
+ HasNewItem = FALSE;
+ PciIo = Private->PciIo;
+
+ //
+ // Submit asynchronous subtasks to the NVMe Submission Queue
+ //
+ for (Link = GetFirstNode (&Private->UnsubmittedSubtasks);
+ !IsNull (&Private->UnsubmittedSubtasks, Link);
+ Link = NextLink)
+ {
+ NextLink = GetNextNode (&Private->UnsubmittedSubtasks, Link);
+ Subtask = NVME_BLKIO2_SUBTASK_FROM_LINK (Link);
+ BlkIo2Request = Subtask->BlockIo2Request;
+ Token = BlkIo2Request->Token;
+ RemoveEntryList (Link);
+ BlkIo2Request->UnsubmittedSubtaskNum--;
+
+ //
+ // If any previous subtask fails, do not process subsequent ones.
+ //
+ if (Token->TransactionStatus != EFI_SUCCESS) {
+ if (IsListEmpty (&BlkIo2Request->SubtasksQueue) &&
+ BlkIo2Request->LastSubtaskSubmitted &&
+ (BlkIo2Request->UnsubmittedSubtaskNum == 0))
+ {
+ //
+ // Remove the BlockIo2 request from the device asynchronous queue.
+ //
+ RemoveEntryList (&BlkIo2Request->Link);
+ FreePool (BlkIo2Request);
+ gBS->SignalEvent (Token->Event);
+ }
+
+ FreePool (Subtask->CommandPacket->NvmeCmd);
+ FreePool (Subtask->CommandPacket->NvmeCompletion);
+ FreePool (Subtask->CommandPacket);
+ FreePool (Subtask);
+
+ continue;
+ }
+
+ Status = Private->Passthru.PassThru (
+ &Private->Passthru,
+ Subtask->NamespaceId,
+ Subtask->CommandPacket,
+ Subtask->Event
+ );
+ if (Status == EFI_NOT_READY) {
+ InsertHeadList (&Private->UnsubmittedSubtasks, Link);
+ BlkIo2Request->UnsubmittedSubtaskNum++;
+ break;
+ } else if (EFI_ERROR (Status)) {
+ Token->TransactionStatus = EFI_DEVICE_ERROR;
+
+ if (IsListEmpty (&BlkIo2Request->SubtasksQueue) &&
+ Subtask->IsLast)
+ {
+ //
+ // Remove the BlockIo2 request from the device asynchronous queue.
+ //
+ RemoveEntryList (&BlkIo2Request->Link);
+ FreePool (BlkIo2Request);
+ gBS->SignalEvent (Token->Event);
+ }
+
+ FreePool (Subtask->CommandPacket->NvmeCmd);
+ FreePool (Subtask->CommandPacket->NvmeCompletion);
+ FreePool (Subtask->CommandPacket);
+ FreePool (Subtask);
+ } else {
+ InsertTailList (&BlkIo2Request->SubtasksQueue, Link);
+ if (Subtask->IsLast) {
+ BlkIo2Request->LastSubtaskSubmitted = TRUE;
+ }
+ }
+ }
+
+ while (Cq->Pt != Private->Pt[QueueId]) {
+ ASSERT (Cq->Sqid == QueueId);
+
+ HasNewItem = TRUE;
+
+ //
+ // Find the command with given Command Id.
+ //
+ for (Link = GetFirstNode (&Private->AsyncPassThruQueue);
+ !IsNull (&Private->AsyncPassThruQueue, Link);
+ Link = NextLink)
+ {
+ NextLink = GetNextNode (&Private->AsyncPassThruQueue, Link);
+ AsyncRequest = NVME_PASS_THRU_ASYNC_REQ_FROM_THIS (Link);
+ if (AsyncRequest->CommandId == Cq->Cid) {
+ //
+ // Copy the Respose Queue entry for this command to the callers
+ // response buffer.
+ //
+ CopyMem (
+ AsyncRequest->Packet->NvmeCompletion,
+ Cq,
+ sizeof (EFI_NVM_EXPRESS_COMPLETION)
+ );
+
+ //
+ // Free the resources allocated before cmd submission
+ //
+ if (AsyncRequest->MapData != NULL) {
+ PciIo->Unmap (PciIo, AsyncRequest->MapData);
+ }
+
+ if (AsyncRequest->MapMeta != NULL) {
+ PciIo->Unmap (PciIo, AsyncRequest->MapMeta);
+ }
+
+ if (AsyncRequest->MapPrpList != NULL) {
+ PciIo->Unmap (PciIo, AsyncRequest->MapPrpList);
+ }
+
+ if (AsyncRequest->PrpListHost != NULL) {
+ PciIo->FreeBuffer (
+ PciIo,
+ AsyncRequest->PrpListNo,
+ AsyncRequest->PrpListHost
+ );
+ }
+
+ RemoveEntryList (Link);
+ gBS->SignalEvent (AsyncRequest->CallerEvent);
+ FreePool (AsyncRequest);
+
+ //
+ // Update submission queue head.
+ //
+ Private->AsyncSqHead = Cq->Sqhd;
+ break;
+ }
+ }
+
+ Private->CqHdbl[QueueId].Cqh++;
+ if (Private->CqHdbl[QueueId].Cqh > MIN (NVME_ASYNC_CCQ_SIZE, Private->Cap.Mqes)) {
+ Private->CqHdbl[QueueId].Cqh = 0;
+ Private->Pt[QueueId] ^= 1;
+ }
+
+ Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
+ }
+
+ if (HasNewItem) {
+ Data = ReadUnaligned32 ((UINT32 *)&Private->CqHdbl[QueueId]);
+ PciIo->Mem.Write (
+ PciIo,
+ EfiPciIoWidthUint32,
+ NVME_BAR,
+ NVME_CQHDBL_OFFSET (QueueId, Private->Cap.Dstrd),
+ 1,
+ &Data
+ );
+ }
+}
+
/**
Tests to see if this driver supports a given controller. If a child device is provided,
it further tests to see if this driver supports creating a handle for the specified child device.
@@ -551,8 +791,9 @@ NvmExpressDriverBindingSupported (
if ((DevicePathNode.DevPath->Type != MESSAGING_DEVICE_PATH) ||
(DevicePathNode.DevPath->SubType != MSG_NVME_NAMESPACE_DP) ||
- (DevicePathNodeLength(DevicePathNode.DevPath) != sizeof(NVME_NAMESPACE_DEVICE_PATH))) {
- return EFI_UNSUPPORTED;
+ (DevicePathNodeLength (DevicePathNode.DevPath) != sizeof (NVME_NAMESPACE_DEVICE_PATH)))
+ {
+ return EFI_UNSUPPORTED;
}
}
}
@@ -563,7 +804,7 @@ NvmExpressDriverBindingSupported (
Status = gBS->OpenProtocol (
Controller,
&gEfiDevicePathProtocolGuid,
- (VOID **) &ParentDevicePath,
+ (VOID **)&ParentDevicePath,
This->DriverBindingHandle,
Controller,
EFI_OPEN_PROTOCOL_BY_DRIVER
@@ -592,7 +833,7 @@ NvmExpressDriverBindingSupported (
Status = gBS->OpenProtocol (
Controller,
&gEfiPciIoProtocolGuid,
- (VOID **) &PciIo,
+ (VOID **)&PciIo,
This->DriverBindingHandle,
Controller,
EFI_OPEN_PROTOCOL_BY_DRIVER
@@ -638,7 +879,6 @@ Done:
return Status;
}
-
/**
Starts a device controller or a bus controller.
@@ -691,7 +931,7 @@ NvmExpressDriverBindingStart (
UINTN Bytes;
EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *Passthru;
- DEBUG ((EFI_D_INFO, "NvmExpressDriverBindingStart: start\n"));
+ DEBUG ((DEBUG_INFO, "NvmExpressDriverBindingStart: start\n"));
Private = NULL;
Passthru = NULL;
@@ -700,7 +940,7 @@ NvmExpressDriverBindingStart (
Status = gBS->OpenProtocol (
Controller,
&gEfiDevicePathProtocolGuid,
- (VOID **) &ParentDevicePath,
+ (VOID **)&ParentDevicePath,
This->DriverBindingHandle,
Controller,
EFI_OPEN_PROTOCOL_BY_DRIVER
@@ -712,7 +952,7 @@ NvmExpressDriverBindingStart (
Status = gBS->OpenProtocol (
Controller,
&gEfiPciIoProtocolGuid,
- (VOID **) &PciIo,
+ (VOID **)&PciIo,
This->DriverBindingHandle,
Controller,
EFI_OPEN_PROTOCOL_BY_DRIVER
@@ -729,33 +969,62 @@ NvmExpressDriverBindingStart (
Private = AllocateZeroPool (sizeof (NVME_CONTROLLER_PRIVATE_DATA));
if (Private == NULL) {
- DEBUG ((EFI_D_ERROR, "NvmExpressDriverBindingStart: allocating pool for Nvme Private Data failed!\n"));
+ DEBUG ((DEBUG_ERROR, "NvmExpressDriverBindingStart: allocating pool for Nvme Private Data failed!\n"));
Status = EFI_OUT_OF_RESOURCES;
goto Exit;
}
//
- // 4 x 4kB aligned buffers will be carved out of this buffer.
+ // Save original PCI attributes
+ //
+ Status = PciIo->Attributes (
+ PciIo,
+ EfiPciIoAttributeOperationGet,
+ 0,
+ &Private->PciAttributes
+ );
+
+ if (EFI_ERROR (Status)) {
+ return Status;
+ }
+
+ //
+ // Enable 64-bit DMA support in the PCI layer.
+ //
+ Status = PciIo->Attributes (
+ PciIo,
+ EfiPciIoAttributeOperationEnable,
+ EFI_PCI_IO_ATTRIBUTE_DUAL_ADDRESS_CYCLE,
+ NULL
+ );
+ if (EFI_ERROR (Status)) {
+ DEBUG ((DEBUG_WARN, "NvmExpressDriverBindingStart: failed to enable 64-bit DMA (%r)\n", Status));
+ }
+
+ //
+ // 6 x 4kB aligned buffers will be carved out of this buffer.
// 1st 4kB boundary is the start of the admin submission queue.
// 2nd 4kB boundary is the start of the admin completion queue.
// 3rd 4kB boundary is the start of I/O submission queue #1.
// 4th 4kB boundary is the start of I/O completion queue #1.
+ // 5th 4kB boundary is the start of I/O submission queue #2.
+ // 6th 4kB boundary is the start of I/O completion queue #2.
//
- // Allocate 4 pages of memory, then map it for bus master read and write.
+ // Allocate 6 pages of memory, then map it for bus master read and write.
//
Status = PciIo->AllocateBuffer (
PciIo,
AllocateAnyPages,
EfiBootServicesData,
- 4,
- (VOID**)&Private->Buffer,
+ 6,
+ (VOID **)&Private->Buffer,
0
);
if (EFI_ERROR (Status)) {
goto Exit;
}
- Bytes = EFI_PAGES_TO_SIZE (4);
+ Bytes = EFI_PAGES_TO_SIZE (6);
Status = PciIo->Map (
PciIo,
EfiPciIoOperationBusMasterCommonBuffer,
@@ -765,13 +1034,13 @@ NvmExpressDriverBindingStart (
&Private->Mapping
);
- if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (4))) {
+ if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (6))) {
goto Exit;
}
Private->BufferPciAddr = (UINT8 *)(UINTN)MappedAddr;
- Private->Signature = NVME_CONTROLLER_PRIVATE_DATA_SIGNATURE;
+ Private->Signature = NVME_CONTROLLER_PRIVATE_DATA_SIGNATURE;
Private->ControllerHandle = Controller;
Private->ImageHandle = This->DriverBindingHandle;
Private->DriverBindingHandle = This->DriverBindingHandle;
@@ -783,9 +1052,34 @@ NvmExpressDriverBindingStart (
Private->Passthru.BuildDevicePath = NvmExpressBuildDevicePath;
Private->Passthru.GetNamespace = NvmExpressGetNamespace;
CopyMem (&Private->PassThruMode, &gEfiNvmExpressPassThruMode, sizeof (EFI_NVM_EXPRESS_PASS_THRU_MODE));
+ InitializeListHead (&Private->AsyncPassThruQueue);
+ InitializeListHead (&Private->UnsubmittedSubtasks);
Status = NvmeControllerInit (Private);
- if (EFI_ERROR(Status)) {
+ if (EFI_ERROR (Status)) {
+ goto Exit;
+ }
+
+ //
+ // Start the asynchronous I/O completion monitor
+ //
+ Status = gBS->CreateEvent (
+ EVT_TIMER | EVT_NOTIFY_SIGNAL,
+ TPL_NOTIFY,
+ ProcessAsyncTaskList,
+ Private,
+ &Private->TimerEvent
+ );
+ if (EFI_ERROR (Status)) {
+ goto Exit;
+ }
+
+ Status = gBS->SetTimer (
+ Private->TimerEvent,
+ TimerPeriodic,
+ NVME_HC_ASYNC_TIMER
+ );
+ if (EFI_ERROR (Status)) {
goto Exit;
}
@@ -798,11 +1092,13 @@ NvmExpressDriverBindingStart (
if (EFI_ERROR (Status)) {
goto Exit;
}
+
+ NvmeRegisterShutdownNotification ();
} else {
Status = gBS->OpenProtocol (
Controller,
&gEfiNvmExpressPassThruProtocolGuid,
- (VOID **) &Passthru,
+ (VOID **)&Passthru,
This->DriverBindingHandle,
Controller,
EFI_OPEN_PROTOCOL_GET_PROTOCOL
@@ -821,7 +1117,6 @@ NvmExpressDriverBindingStart (
Status = DiscoverAllNamespaces (
Private
);
-
} else if (!IsDevicePathEnd (RemainingDevicePath)) {
//
// Enumerate the specified NVME namespace
@@ -840,7 +1135,7 @@ NvmExpressDriverBindingStart (
}
}
- DEBUG ((EFI_D_INFO, "NvmExpressDriverBindingStart: end successfully\n"));
+ DEBUG ((DEBUG_INFO, "NvmExpressDriverBindingStart: end successfully\n"));
return EFI_SUCCESS;
Exit:
@@ -849,7 +1144,7 @@ Exit:
}
if ((Private != NULL) && (Private->Buffer != NULL)) {
- PciIo->FreeBuffer (PciIo, 4, Private->Buffer);
+ PciIo->FreeBuffer (PciIo, 6, Private->Buffer);
}
if ((Private != NULL) && (Private->ControllerData != NULL)) {
@@ -857,6 +1152,10 @@ Exit:
}
if (Private != NULL) {
+ if (Private->TimerEvent != NULL) {
+ gBS->CloseEvent (Private->TimerEvent);
+ }
+
FreePool (Private);
}
@@ -874,12 +1173,11 @@ Exit:
Controller
);
- DEBUG ((EFI_D_INFO, "NvmExpressDriverBindingStart: end with %r\n", Status));
+ DEBUG ((DEBUG_INFO, "NvmExpressDriverBindingStart: end with %r\n", Status));
return Status;
}
-
/**
Stops a device controller or a bus controller.
@@ -909,10 +1207,10 @@ Exit:
EFI_STATUS
EFIAPI
NvmExpressDriverBindingStop (
- IN EFI_DRIVER_BINDING_PROTOCOL *This,
- IN EFI_HANDLE Controller,
- IN UINTN NumberOfChildren,
- IN EFI_HANDLE *ChildHandleBuffer
+ IN EFI_DRIVER_BINDING_PROTOCOL *This,
+ IN EFI_HANDLE Controller,
+ IN UINTN NumberOfChildren,
+ IN EFI_HANDLE *ChildHandleBuffer
)
{
EFI_STATUS Status;
@@ -920,12 +1218,14 @@ NvmExpressDriverBindingStop (
UINTN Index;
NVME_CONTROLLER_PRIVATE_DATA *Private;
EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *PassThru;
+ BOOLEAN IsEmpty;
+ EFI_TPL OldTpl;
if (NumberOfChildren == 0) {
Status = gBS->OpenProtocol (
Controller,
&gEfiNvmExpressPassThruProtocolGuid,
- (VOID **) &PassThru,
+ (VOID **)&PassThru,
This->DriverBindingHandle,
Controller,
EFI_OPEN_PROTOCOL_GET_PROTOCOL
@@ -933,19 +1233,40 @@ NvmExpressDriverBindingStop (
if (!EFI_ERROR (Status)) {
Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (PassThru);
+
+ //
+ // Wait for the asynchronous PassThru queue to become empty.
+ //
+ while (TRUE) {
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);
+ IsEmpty = IsListEmpty (&Private->AsyncPassThruQueue) &&
+ IsListEmpty (&Private->UnsubmittedSubtasks);
+ gBS->RestoreTPL (OldTpl);
+
+ if (IsEmpty) {
+ break;
+ }
+
+ gBS->Stall (100);
+ }
+
gBS->UninstallMultipleProtocolInterfaces (
- Controller,
- &gEfiNvmExpressPassThruProtocolGuid,
- PassThru,
- NULL
- );
+ Controller,
+ &gEfiNvmExpressPassThruProtocolGuid,
+ PassThru,
+ NULL
+ );
+
+ if (Private->TimerEvent != NULL) {
+ gBS->CloseEvent (Private->TimerEvent);
+ }
if (Private->Mapping != NULL) {
Private->PciIo->Unmap (Private->PciIo, Private->Mapping);
}
if (Private->Buffer != NULL) {
- Private->PciIo->FreeBuffer (Private->PciIo, 4, Private->Buffer);
+ Private->PciIo->FreeBuffer (Private->PciIo, 6, Private->Buffer);
}
FreePool (Private->ControllerData);
@@ -953,17 +1274,20 @@ NvmExpressDriverBindingStop (
}
gBS->CloseProtocol (
- Controller,
- &gEfiPciIoProtocolGuid,
- This->DriverBindingHandle,
- Controller
- );
+ Controller,
+ &gEfiPciIoProtocolGuid,
+ This->DriverBindingHandle,
+ Controller
+ );
gBS->CloseProtocol (
- Controller,
- &gEfiDevicePathProtocolGuid,
- This->DriverBindingHandle,
- Controller
- );
+ Controller,
+ &gEfiDevicePathProtocolGuid,
+ This->DriverBindingHandle,
+ Controller
+ );
+
+ NvmeUnregisterShutdownNotification ();
+
return EFI_SUCCESS;
}
@@ -998,15 +1322,15 @@ NvmExpressDriverBindingStop (
EFI_STATUS
EFIAPI
NvmExpressUnload (
- IN EFI_HANDLE ImageHandle
+ IN EFI_HANDLE ImageHandle
)
{
- EFI_STATUS Status;
- EFI_HANDLE *DeviceHandleBuffer;
- UINTN DeviceHandleCount;
- UINTN Index;
- EFI_COMPONENT_NAME_PROTOCOL *ComponentName;
- EFI_COMPONENT_NAME2_PROTOCOL *ComponentName2;
+ EFI_STATUS Status;
+ EFI_HANDLE *DeviceHandleBuffer;
+ UINTN DeviceHandleCount;
+ UINTN Index;
+ EFI_COMPONENT_NAME_PROTOCOL *ComponentName;
+ EFI_COMPONENT_NAME2_PROTOCOL *ComponentName2;
//
// Get the list of the device handles managed by this driver.
@@ -1015,13 +1339,13 @@ NvmExpressUnload (
// those protocols installed at image handle.
//
DeviceHandleBuffer = NULL;
- Status = gBS->LocateHandleBuffer (
- ByProtocol,
- &gEfiNvmExpressPassThruProtocolGuid,
- NULL,
- &DeviceHandleCount,
- &DeviceHandleBuffer
- );
+ Status = gBS->LocateHandleBuffer (
+ ByProtocol,
+ &gEfiNvmExpressPassThruProtocolGuid,
+ NULL,
+ &DeviceHandleCount,
+ &DeviceHandleBuffer
+ );
if (!EFI_ERROR (Status)) {
//
@@ -1068,7 +1392,7 @@ NvmExpressUnload (
Status = gBS->HandleProtocol (
ImageHandle,
&gEfiComponentNameProtocolGuid,
- (VOID **) &ComponentName
+ (VOID **)&ComponentName
);
if (!EFI_ERROR (Status)) {
gBS->UninstallProtocolInterface (
@@ -1081,7 +1405,7 @@ NvmExpressUnload (
Status = gBS->HandleProtocol (
ImageHandle,
&gEfiComponentName2ProtocolGuid,
- (VOID **) &ComponentName2
+ (VOID **)&ComponentName2
);
if (!EFI_ERROR (Status)) {
gBS->UninstallProtocolInterface (
@@ -1100,6 +1424,7 @@ EXIT:
if (DeviceHandleBuffer != NULL) {
gBS->FreePool (DeviceHandleBuffer);
}
+
return Status;
}
@@ -1120,7 +1445,7 @@ NvmExpressDriverEntry (
IN EFI_SYSTEM_TABLE *SystemTable
)
{
- EFI_STATUS Status;
+ EFI_STATUS Status;
Status = EfiLibInstallDriverBindingComponentName2 (
ImageHandle,
@@ -1137,12 +1462,12 @@ NvmExpressDriverEntry (
// EFI drivers that are on PCI and other plug in cards.
//
gNvmExpressDriverSupportedEfiVersion.FirmwareVersion = 0x00020028;
- Status = gBS->InstallMultipleProtocolInterfaces (
- &ImageHandle,
- &gEfiDriverSupportedEfiVersionProtocolGuid,
- &gNvmExpressDriverSupportedEfiVersion,
- NULL
- );
+ Status = gBS->InstallMultipleProtocolInterfaces (
+ &ImageHandle,
+ &gEfiDriverSupportedEfiVersionProtocolGuid,
+ &gNvmExpressDriverSupportedEfiVersion,
+ NULL
+ );
ASSERT_EFI_ERROR (Status);
return Status;
}