NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows\r
NVM Express specification.\r
\r
- Copyright (c) 2013 - 2016, Intel Corporation. All rights reserved.<BR>\r
- This program and the accompanying materials\r
- are licensed and made available under the terms and conditions of the BSD License\r
- which accompanies this distribution. The full text of the license may be found at\r
- http://opensource.org/licenses/bsd-license.php.\r
-\r
- THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,\r
- WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.\r
+ Copyright (c) 2013 - 2017, Intel Corporation. All rights reserved.<BR>\r
+ SPDX-License-Identifier: BSD-2-Clause-Patent\r
\r
**/\r
\r
//\r
// NVM Express Driver Binding Protocol Instance\r
//\r
-EFI_DRIVER_BINDING_PROTOCOL gNvmExpressDriverBinding = {\r
+EFI_DRIVER_BINDING_PROTOCOL gNvmExpressDriverBinding = {\r
NvmExpressDriverBindingSupported,\r
NvmExpressDriverBindingStart,\r
NvmExpressDriverBindingStop,\r
//\r
// NVM Express EFI Driver Supported EFI Version Protocol Instance\r
//\r
-EFI_DRIVER_SUPPORTED_EFI_VERSION_PROTOCOL gNvmExpressDriverSupportedEfiVersion = {\r
+EFI_DRIVER_SUPPORTED_EFI_VERSION_PROTOCOL gNvmExpressDriverSupportedEfiVersion = {\r
sizeof (EFI_DRIVER_SUPPORTED_EFI_VERSION_PROTOCOL), // Size of Protocol structure.\r
0 // Version number to be filled at start up.\r
};\r
//\r
// Template for NVM Express Pass Thru Mode data structure.\r
//\r
-GLOBAL_REMOVE_IF_UNREFERENCED EFI_NVM_EXPRESS_PASS_THRU_MODE gEfiNvmExpressPassThruMode = {\r
- EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL | EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL | EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_CMD_SET_NVM,\r
+GLOBAL_REMOVE_IF_UNREFERENCED EFI_NVM_EXPRESS_PASS_THRU_MODE gEfiNvmExpressPassThruMode = {\r
+ EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL |\r
+ EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL |\r
+ EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_NONBLOCKIO |\r
+ EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_CMD_SET_NVM,\r
sizeof (UINTN),\r
0x10100\r
};\r
**/\r
EFI_STATUS\r
EnumerateNvmeDevNamespace (\r
- IN NVME_CONTROLLER_PRIVATE_DATA *Private,\r
- UINT32 NamespaceId\r
+ IN NVME_CONTROLLER_PRIVATE_DATA *Private,\r
+ UINT32 NamespaceId\r
)\r
{\r
- NVME_ADMIN_NAMESPACE_DATA *NamespaceData;\r
- EFI_DEVICE_PATH_PROTOCOL *NewDevicePathNode;\r
- EFI_DEVICE_PATH_PROTOCOL *DevicePath;\r
- EFI_HANDLE DeviceHandle;\r
- EFI_DEVICE_PATH_PROTOCOL *ParentDevicePath;\r
- EFI_DEVICE_PATH_PROTOCOL *RemainingDevicePath;\r
- NVME_DEVICE_PRIVATE_DATA *Device;\r
- EFI_STATUS Status;\r
- UINT32 Lbads;\r
- UINT32 Flbas;\r
- UINT32 LbaFmtIdx;\r
- UINT8 Sn[21];\r
- UINT8 Mn[41];\r
+ NVME_ADMIN_NAMESPACE_DATA *NamespaceData;\r
+ EFI_DEVICE_PATH_PROTOCOL *NewDevicePathNode;\r
+ EFI_DEVICE_PATH_PROTOCOL *DevicePath;\r
+ EFI_HANDLE DeviceHandle;\r
+ EFI_DEVICE_PATH_PROTOCOL *ParentDevicePath;\r
+ EFI_DEVICE_PATH_PROTOCOL *RemainingDevicePath;\r
+ NVME_DEVICE_PRIVATE_DATA *Device;\r
+ EFI_STATUS Status;\r
+ UINT32 Lbads;\r
+ UINT32 Flbas;\r
+ UINT32 LbaFmtIdx;\r
+ UINT8 Sn[21];\r
+ UINT8 Mn[41];\r
+ VOID *DummyInterface;\r
\r
NewDevicePathNode = NULL;\r
DevicePath = NULL;\r
//\r
// Allocate a buffer for Identify Namespace data\r
//\r
- NamespaceData = AllocateZeroPool(sizeof (NVME_ADMIN_NAMESPACE_DATA));\r
- if(NamespaceData == NULL) {\r
+ NamespaceData = AllocateZeroPool (sizeof (NVME_ADMIN_NAMESPACE_DATA));\r
+ if (NamespaceData == NULL) {\r
return EFI_OUT_OF_RESOURCES;\r
}\r
\r
NamespaceId,\r
(VOID *)NamespaceData\r
);\r
- if (EFI_ERROR(Status)) {\r
+ if (EFI_ERROR (Status)) {\r
goto Exit;\r
}\r
+\r
//\r
// Validate Namespace\r
//\r
//\r
// allocate device private data for each discovered namespace\r
//\r
- Device = AllocateZeroPool(sizeof(NVME_DEVICE_PRIVATE_DATA));\r
+ Device = AllocateZeroPool (sizeof (NVME_DEVICE_PRIVATE_DATA));\r
if (Device == NULL) {\r
Status = EFI_OUT_OF_RESOURCES;\r
goto Exit;\r
//\r
// Initialize SSD namespace instance data\r
//\r
- Device->Signature = NVME_DEVICE_PRIVATE_DATA_SIGNATURE;\r
- Device->NamespaceId = NamespaceId;\r
- Device->NamespaceUuid = NamespaceData->Eui64;\r
+ Device->Signature = NVME_DEVICE_PRIVATE_DATA_SIGNATURE;\r
+ Device->NamespaceId = NamespaceId;\r
+ Device->NamespaceUuid = NamespaceData->Eui64;\r
\r
Device->ControllerHandle = Private->ControllerHandle;\r
Device->DriverBindingHandle = Private->DriverBindingHandle;\r
//\r
// Build BlockIo media structure\r
//\r
- Device->Media.MediaId = 0;\r
- Device->Media.RemovableMedia = FALSE;\r
- Device->Media.MediaPresent = TRUE;\r
+ Device->Media.MediaId = 0;\r
+ Device->Media.RemovableMedia = FALSE;\r
+ Device->Media.MediaPresent = TRUE;\r
Device->Media.LogicalPartition = FALSE;\r
- Device->Media.ReadOnly = FALSE;\r
- Device->Media.WriteCaching = FALSE;\r
+ Device->Media.ReadOnly = FALSE;\r
+ Device->Media.WriteCaching = FALSE;\r
+ Device->Media.IoAlign = Private->PassThruMode.IoAlign;\r
\r
Flbas = NamespaceData->Flbas;\r
LbaFmtIdx = Flbas & 0xF;\r
- Lbads = NamespaceData->LbaFormat[LbaFmtIdx].Lbads;\r
+\r
+ //\r
+ // Currently this NVME driver only suport Metadata Size == 0\r
+ //\r
+ if (NamespaceData->LbaFormat[LbaFmtIdx].Ms != 0) {\r
+ DEBUG ((\r
+ DEBUG_ERROR,\r
+ "NVME IDENTIFY NAMESPACE [%d] Ms(%d) is not supported.\n",\r
+ NamespaceId,\r
+ NamespaceData->LbaFormat[LbaFmtIdx].Ms\r
+ ));\r
+ Status = EFI_UNSUPPORTED;\r
+ goto Exit;\r
+ }\r
+\r
+ Lbads = NamespaceData->LbaFormat[LbaFmtIdx].Lbads;\r
Device->Media.BlockSize = (UINT32)1 << Lbads;\r
\r
Device->Media.LastBlock = NamespaceData->Nsze - 1;\r
//\r
// Create BlockIo Protocol instance\r
//\r
- Device->BlockIo.Revision = EFI_BLOCK_IO_PROTOCOL_REVISION2;\r
- Device->BlockIo.Media = &Device->Media;\r
- Device->BlockIo.Reset = NvmeBlockIoReset;\r
- Device->BlockIo.ReadBlocks = NvmeBlockIoReadBlocks;\r
- Device->BlockIo.WriteBlocks = NvmeBlockIoWriteBlocks;\r
- Device->BlockIo.FlushBlocks = NvmeBlockIoFlushBlocks;\r
+ Device->BlockIo.Revision = EFI_BLOCK_IO_PROTOCOL_REVISION2;\r
+ Device->BlockIo.Media = &Device->Media;\r
+ Device->BlockIo.Reset = NvmeBlockIoReset;\r
+ Device->BlockIo.ReadBlocks = NvmeBlockIoReadBlocks;\r
+ Device->BlockIo.WriteBlocks = NvmeBlockIoWriteBlocks;\r
+ Device->BlockIo.FlushBlocks = NvmeBlockIoFlushBlocks;\r
+\r
+ //\r
+ // Create BlockIo2 Protocol instance\r
+ //\r
+ Device->BlockIo2.Media = &Device->Media;\r
+ Device->BlockIo2.Reset = NvmeBlockIoResetEx;\r
+ Device->BlockIo2.ReadBlocksEx = NvmeBlockIoReadBlocksEx;\r
+ Device->BlockIo2.WriteBlocksEx = NvmeBlockIoWriteBlocksEx;\r
+ Device->BlockIo2.FlushBlocksEx = NvmeBlockIoFlushBlocksEx;\r
+ InitializeListHead (&Device->AsyncQueue);\r
\r
//\r
// Create StorageSecurityProtocol Instance\r
&NewDevicePathNode\r
);\r
\r
- if (EFI_ERROR(Status)) {\r
+ if (EFI_ERROR (Status)) {\r
goto Exit;\r
}\r
\r
goto Exit;\r
}\r
\r
- DeviceHandle = NULL;\r
+ DeviceHandle = NULL;\r
RemainingDevicePath = DevicePath;\r
- Status = gBS->LocateDevicePath (&gEfiDevicePathProtocolGuid, &RemainingDevicePath, &DeviceHandle);\r
- if (!EFI_ERROR (Status) && (DeviceHandle != NULL) && IsDevicePathEnd(RemainingDevicePath)) {\r
+ Status = gBS->LocateDevicePath (&gEfiDevicePathProtocolGuid, &RemainingDevicePath, &DeviceHandle);\r
+ if (!EFI_ERROR (Status) && (DeviceHandle != NULL) && IsDevicePathEnd (RemainingDevicePath)) {\r
Status = EFI_ALREADY_STARTED;\r
FreePool (DevicePath);\r
goto Exit;\r
Device->DevicePath,\r
&gEfiBlockIoProtocolGuid,\r
&Device->BlockIo,\r
+ &gEfiBlockIo2ProtocolGuid,\r
+ &Device->BlockIo2,\r
&gEfiDiskInfoProtocolGuid,\r
&Device->DiskInfo,\r
NULL\r
);\r
\r
- if(EFI_ERROR(Status)) {\r
+ if (EFI_ERROR (Status)) {\r
goto Exit;\r
}\r
\r
EFI_NATIVE_INTERFACE,\r
&Device->StorageSecurity\r
);\r
- if(EFI_ERROR(Status)) {\r
+ if (EFI_ERROR (Status)) {\r
gBS->UninstallMultipleProtocolInterfaces (\r
- &Device->DeviceHandle,\r
+ Device->DeviceHandle,\r
&gEfiDevicePathProtocolGuid,\r
Device->DevicePath,\r
&gEfiBlockIoProtocolGuid,\r
&Device->BlockIo,\r
+ &gEfiBlockIo2ProtocolGuid,\r
+ &Device->BlockIo2,\r
&gEfiDiskInfoProtocolGuid,\r
&Device->DiskInfo,\r
NULL\r
gBS->OpenProtocol (\r
Private->ControllerHandle,\r
&gEfiNvmExpressPassThruProtocolGuid,\r
- (VOID **) &Private->Passthru,\r
+ (VOID **)&DummyInterface,\r
Private->DriverBindingHandle,\r
Device->DeviceHandle,\r
EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER\r
//\r
// Dump NvmExpress Identify Namespace Data\r
//\r
- DEBUG ((EFI_D_INFO, " == NVME IDENTIFY NAMESPACE [%d] DATA ==\n", NamespaceId));\r
- DEBUG ((EFI_D_INFO, " NSZE : 0x%x\n", NamespaceData->Nsze));\r
- DEBUG ((EFI_D_INFO, " NCAP : 0x%x\n", NamespaceData->Ncap));\r
- DEBUG ((EFI_D_INFO, " NUSE : 0x%x\n", NamespaceData->Nuse));\r
- DEBUG ((EFI_D_INFO, " LBAF0.LBADS : 0x%x\n", (NamespaceData->LbaFormat[0].Lbads)));\r
+ DEBUG ((DEBUG_INFO, " == NVME IDENTIFY NAMESPACE [%d] DATA ==\n", NamespaceId));\r
+ DEBUG ((DEBUG_INFO, " NSZE : 0x%x\n", NamespaceData->Nsze));\r
+ DEBUG ((DEBUG_INFO, " NCAP : 0x%x\n", NamespaceData->Ncap));\r
+ DEBUG ((DEBUG_INFO, " NUSE : 0x%x\n", NamespaceData->Nuse));\r
+ DEBUG ((DEBUG_INFO, " LBAF0.LBADS : 0x%x\n", (NamespaceData->LbaFormat[0].Lbads)));\r
\r
//\r
// Build controller name for Component Name (2) protocol.\r
}\r
\r
Exit:\r
- if(NamespaceData != NULL) {\r
+ if (NamespaceData != NULL) {\r
FreePool (NamespaceData);\r
}\r
\r
FreePool (NewDevicePathNode);\r
}\r
\r
- if(EFI_ERROR(Status) && (Device != NULL) && (Device->DevicePath != NULL)) {\r
+ if (EFI_ERROR (Status) && (Device != NULL) && (Device->DevicePath != NULL)) {\r
FreePool (Device->DevicePath);\r
}\r
- if(EFI_ERROR(Status) && (Device != NULL)) {\r
+\r
+ if (EFI_ERROR (Status) && (Device != NULL)) {\r
FreePool (Device);\r
}\r
+\r
return Status;\r
}\r
\r
**/\r
EFI_STATUS\r
DiscoverAllNamespaces (\r
- IN NVME_CONTROLLER_PRIVATE_DATA *Private\r
+ IN NVME_CONTROLLER_PRIVATE_DATA *Private\r
)\r
{\r
- EFI_STATUS Status;\r
- UINT32 NamespaceId;\r
- EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *Passthru;\r
+ EFI_STATUS Status;\r
+ UINT32 NamespaceId;\r
+ EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *Passthru;\r
\r
- NamespaceId = 0xFFFFFFFF;\r
- Passthru = &Private->Passthru;\r
+ NamespaceId = 0xFFFFFFFF;\r
+ Passthru = &Private->Passthru;\r
\r
while (TRUE) {\r
Status = Passthru->GetNextNamespace (\r
NamespaceId\r
);\r
\r
- if (EFI_ERROR(Status)) {\r
+ if (EFI_ERROR (Status)) {\r
continue;\r
}\r
}\r
**/\r
EFI_STATUS\r
UnregisterNvmeNamespace (\r
- IN EFI_DRIVER_BINDING_PROTOCOL *This,\r
- IN EFI_HANDLE Controller,\r
- IN EFI_HANDLE Handle\r
+ IN EFI_DRIVER_BINDING_PROTOCOL *This,\r
+ IN EFI_HANDLE Controller,\r
+ IN EFI_HANDLE Handle\r
)\r
{\r
- EFI_STATUS Status;\r
- EFI_BLOCK_IO_PROTOCOL *BlockIo;\r
- NVME_DEVICE_PRIVATE_DATA *Device;\r
- NVME_CONTROLLER_PRIVATE_DATA *Private;\r
- EFI_STORAGE_SECURITY_COMMAND_PROTOCOL *StorageSecurity;\r
+ EFI_STATUS Status;\r
+ EFI_BLOCK_IO_PROTOCOL *BlockIo;\r
+ NVME_DEVICE_PRIVATE_DATA *Device;\r
+ EFI_STORAGE_SECURITY_COMMAND_PROTOCOL *StorageSecurity;\r
+ BOOLEAN IsEmpty;\r
+ EFI_TPL OldTpl;\r
+ VOID *DummyInterface;\r
\r
BlockIo = NULL;\r
\r
Status = gBS->OpenProtocol (\r
Handle,\r
&gEfiBlockIoProtocolGuid,\r
- (VOID **) &BlockIo,\r
+ (VOID **)&BlockIo,\r
This->DriverBindingHandle,\r
Controller,\r
EFI_OPEN_PROTOCOL_GET_PROTOCOL\r
return Status;\r
}\r
\r
- Device = NVME_DEVICE_PRIVATE_DATA_FROM_BLOCK_IO (BlockIo);\r
- Private = Device->Controller;\r
+ Device = NVME_DEVICE_PRIVATE_DATA_FROM_BLOCK_IO (BlockIo);\r
+\r
+ //\r
+ // Wait for the device's asynchronous I/O queue to become empty.\r
+ //\r
+ while (TRUE) {\r
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);\r
+ IsEmpty = IsListEmpty (&Device->AsyncQueue);\r
+ gBS->RestoreTPL (OldTpl);\r
+\r
+ if (IsEmpty) {\r
+ break;\r
+ }\r
+\r
+ gBS->Stall (100);\r
+ }\r
\r
//\r
// Close the child handle\r
Device->DevicePath,\r
&gEfiBlockIoProtocolGuid,\r
&Device->BlockIo,\r
+ &gEfiBlockIo2ProtocolGuid,\r
+ &Device->BlockIo2,\r
&gEfiDiskInfoProtocolGuid,\r
&Device->DiskInfo,\r
NULL\r
gBS->OpenProtocol (\r
Controller,\r
&gEfiNvmExpressPassThruProtocolGuid,\r
- (VOID **) &Private->Passthru,\r
+ (VOID **)&DummyInterface,\r
This->DriverBindingHandle,\r
Handle,\r
EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER\r
Status = gBS->OpenProtocol (\r
Handle,\r
&gEfiStorageSecurityCommandProtocolGuid,\r
- (VOID **) &StorageSecurity,\r
+ (VOID **)&StorageSecurity,\r
This->DriverBindingHandle,\r
Controller,\r
EFI_OPEN_PROTOCOL_GET_PROTOCOL\r
);\r
if (EFI_ERROR (Status)) {\r
gBS->OpenProtocol (\r
- Controller,\r
- &gEfiNvmExpressPassThruProtocolGuid,\r
- (VOID **) &Private->Passthru,\r
- This->DriverBindingHandle,\r
- Handle,\r
- EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER\r
- );\r
+ Controller,\r
+ &gEfiNvmExpressPassThruProtocolGuid,\r
+ (VOID **)&DummyInterface,\r
+ This->DriverBindingHandle,\r
+ Handle,\r
+ EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER\r
+ );\r
return Status;\r
}\r
}\r
\r
- if(Device->DevicePath != NULL) {\r
+ if (Device->DevicePath != NULL) {\r
FreePool (Device->DevicePath);\r
}\r
\r
return EFI_SUCCESS;\r
}\r
\r
+/**\r
+ Call back function when the timer event is signaled.\r
+\r
+ @param[in] Event The Event this notify function registered to.\r
+ @param[in] Context Pointer to the context data registered to the\r
+ Event.\r
+\r
+**/\r
+VOID\r
+EFIAPI\r
+ProcessAsyncTaskList (\r
+ IN EFI_EVENT Event,\r
+ IN VOID *Context\r
+ )\r
+{\r
+ NVME_CONTROLLER_PRIVATE_DATA *Private;\r
+ EFI_PCI_IO_PROTOCOL *PciIo;\r
+ NVME_CQ *Cq;\r
+ UINT16 QueueId;\r
+ UINT32 Data;\r
+ LIST_ENTRY *Link;\r
+ LIST_ENTRY *NextLink;\r
+ NVME_PASS_THRU_ASYNC_REQ *AsyncRequest;\r
+ NVME_BLKIO2_SUBTASK *Subtask;\r
+ NVME_BLKIO2_REQUEST *BlkIo2Request;\r
+ EFI_BLOCK_IO2_TOKEN *Token;\r
+ BOOLEAN HasNewItem;\r
+ EFI_STATUS Status;\r
+\r
+ Private = (NVME_CONTROLLER_PRIVATE_DATA *)Context;\r
+ QueueId = 2;\r
+ Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;\r
+ HasNewItem = FALSE;\r
+ PciIo = Private->PciIo;\r
+\r
+ //\r
+ // Submit asynchronous subtasks to the NVMe Submission Queue\r
+ //\r
+ for (Link = GetFirstNode (&Private->UnsubmittedSubtasks);\r
+ !IsNull (&Private->UnsubmittedSubtasks, Link);\r
+ Link = NextLink)\r
+ {\r
+ NextLink = GetNextNode (&Private->UnsubmittedSubtasks, Link);\r
+ Subtask = NVME_BLKIO2_SUBTASK_FROM_LINK (Link);\r
+ BlkIo2Request = Subtask->BlockIo2Request;\r
+ Token = BlkIo2Request->Token;\r
+ RemoveEntryList (Link);\r
+ BlkIo2Request->UnsubmittedSubtaskNum--;\r
+\r
+ //\r
+ // If any previous subtask fails, do not process subsequent ones.\r
+ //\r
+ if (Token->TransactionStatus != EFI_SUCCESS) {\r
+ if (IsListEmpty (&BlkIo2Request->SubtasksQueue) &&\r
+ BlkIo2Request->LastSubtaskSubmitted &&\r
+ (BlkIo2Request->UnsubmittedSubtaskNum == 0))\r
+ {\r
+ //\r
+ // Remove the BlockIo2 request from the device asynchronous queue.\r
+ //\r
+ RemoveEntryList (&BlkIo2Request->Link);\r
+ FreePool (BlkIo2Request);\r
+ gBS->SignalEvent (Token->Event);\r
+ }\r
+\r
+ FreePool (Subtask->CommandPacket->NvmeCmd);\r
+ FreePool (Subtask->CommandPacket->NvmeCompletion);\r
+ FreePool (Subtask->CommandPacket);\r
+ FreePool (Subtask);\r
+\r
+ continue;\r
+ }\r
+\r
+ Status = Private->Passthru.PassThru (\r
+ &Private->Passthru,\r
+ Subtask->NamespaceId,\r
+ Subtask->CommandPacket,\r
+ Subtask->Event\r
+ );\r
+ if (Status == EFI_NOT_READY) {\r
+ InsertHeadList (&Private->UnsubmittedSubtasks, Link);\r
+ BlkIo2Request->UnsubmittedSubtaskNum++;\r
+ break;\r
+ } else if (EFI_ERROR (Status)) {\r
+ Token->TransactionStatus = EFI_DEVICE_ERROR;\r
+\r
+ if (IsListEmpty (&BlkIo2Request->SubtasksQueue) &&\r
+ Subtask->IsLast)\r
+ {\r
+ //\r
+ // Remove the BlockIo2 request from the device asynchronous queue.\r
+ //\r
+ RemoveEntryList (&BlkIo2Request->Link);\r
+ FreePool (BlkIo2Request);\r
+ gBS->SignalEvent (Token->Event);\r
+ }\r
+\r
+ FreePool (Subtask->CommandPacket->NvmeCmd);\r
+ FreePool (Subtask->CommandPacket->NvmeCompletion);\r
+ FreePool (Subtask->CommandPacket);\r
+ FreePool (Subtask);\r
+ } else {\r
+ InsertTailList (&BlkIo2Request->SubtasksQueue, Link);\r
+ if (Subtask->IsLast) {\r
+ BlkIo2Request->LastSubtaskSubmitted = TRUE;\r
+ }\r
+ }\r
+ }\r
+\r
+ while (Cq->Pt != Private->Pt[QueueId]) {\r
+ ASSERT (Cq->Sqid == QueueId);\r
+\r
+ HasNewItem = TRUE;\r
+\r
+ //\r
+ // Find the command with given Command Id.\r
+ //\r
+ for (Link = GetFirstNode (&Private->AsyncPassThruQueue);\r
+ !IsNull (&Private->AsyncPassThruQueue, Link);\r
+ Link = NextLink)\r
+ {\r
+ NextLink = GetNextNode (&Private->AsyncPassThruQueue, Link);\r
+ AsyncRequest = NVME_PASS_THRU_ASYNC_REQ_FROM_THIS (Link);\r
+ if (AsyncRequest->CommandId == Cq->Cid) {\r
+ //\r
+ // Copy the Respose Queue entry for this command to the callers\r
+ // response buffer.\r
+ //\r
+ CopyMem (\r
+ AsyncRequest->Packet->NvmeCompletion,\r
+ Cq,\r
+ sizeof (EFI_NVM_EXPRESS_COMPLETION)\r
+ );\r
+\r
+ //\r
+ // Free the resources allocated before cmd submission\r
+ //\r
+ if (AsyncRequest->MapData != NULL) {\r
+ PciIo->Unmap (PciIo, AsyncRequest->MapData);\r
+ }\r
+\r
+ if (AsyncRequest->MapMeta != NULL) {\r
+ PciIo->Unmap (PciIo, AsyncRequest->MapMeta);\r
+ }\r
+\r
+ if (AsyncRequest->MapPrpList != NULL) {\r
+ PciIo->Unmap (PciIo, AsyncRequest->MapPrpList);\r
+ }\r
+\r
+ if (AsyncRequest->PrpListHost != NULL) {\r
+ PciIo->FreeBuffer (\r
+ PciIo,\r
+ AsyncRequest->PrpListNo,\r
+ AsyncRequest->PrpListHost\r
+ );\r
+ }\r
+\r
+ RemoveEntryList (Link);\r
+ gBS->SignalEvent (AsyncRequest->CallerEvent);\r
+ FreePool (AsyncRequest);\r
+\r
+ //\r
+ // Update submission queue head.\r
+ //\r
+ Private->AsyncSqHead = Cq->Sqhd;\r
+ break;\r
+ }\r
+ }\r
+\r
+ Private->CqHdbl[QueueId].Cqh++;\r
+ if (Private->CqHdbl[QueueId].Cqh > MIN (NVME_ASYNC_CCQ_SIZE, Private->Cap.Mqes)) {\r
+ Private->CqHdbl[QueueId].Cqh = 0;\r
+ Private->Pt[QueueId] ^= 1;\r
+ }\r
+\r
+ Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;\r
+ }\r
+\r
+ if (HasNewItem) {\r
+ Data = ReadUnaligned32 ((UINT32 *)&Private->CqHdbl[QueueId]);\r
+ PciIo->Mem.Write (\r
+ PciIo,\r
+ EfiPciIoWidthUint32,\r
+ NVME_BAR,\r
+ NVME_CQHDBL_OFFSET (QueueId, Private->Cap.Dstrd),\r
+ 1,\r
+ &Data\r
+ );\r
+ }\r
+}\r
+\r
/**\r
Tests to see if this driver supports a given controller. If a child device is provided,\r
it further tests to see if this driver supports creating a handle for the specified child device.\r
\r
if ((DevicePathNode.DevPath->Type != MESSAGING_DEVICE_PATH) ||\r
(DevicePathNode.DevPath->SubType != MSG_NVME_NAMESPACE_DP) ||\r
- (DevicePathNodeLength(DevicePathNode.DevPath) != sizeof(NVME_NAMESPACE_DEVICE_PATH))) {\r
- return EFI_UNSUPPORTED;\r
+ (DevicePathNodeLength (DevicePathNode.DevPath) != sizeof (NVME_NAMESPACE_DEVICE_PATH)))\r
+ {\r
+ return EFI_UNSUPPORTED;\r
}\r
}\r
}\r
Status = gBS->OpenProtocol (\r
Controller,\r
&gEfiDevicePathProtocolGuid,\r
- (VOID **) &ParentDevicePath,\r
+ (VOID **)&ParentDevicePath,\r
This->DriverBindingHandle,\r
Controller,\r
EFI_OPEN_PROTOCOL_BY_DRIVER\r
Status = gBS->OpenProtocol (\r
Controller,\r
&gEfiPciIoProtocolGuid,\r
- (VOID **) &PciIo,\r
+ (VOID **)&PciIo,\r
This->DriverBindingHandle,\r
Controller,\r
EFI_OPEN_PROTOCOL_BY_DRIVER\r
return Status;\r
}\r
\r
-\r
/**\r
Starts a device controller or a bus controller.\r
\r
UINTN Bytes;\r
EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *Passthru;\r
\r
- DEBUG ((EFI_D_INFO, "NvmExpressDriverBindingStart: start\n"));\r
+ DEBUG ((DEBUG_INFO, "NvmExpressDriverBindingStart: start\n"));\r
\r
Private = NULL;\r
Passthru = NULL;\r
Status = gBS->OpenProtocol (\r
Controller,\r
&gEfiDevicePathProtocolGuid,\r
- (VOID **) &ParentDevicePath,\r
+ (VOID **)&ParentDevicePath,\r
This->DriverBindingHandle,\r
Controller,\r
EFI_OPEN_PROTOCOL_BY_DRIVER\r
Status = gBS->OpenProtocol (\r
Controller,\r
&gEfiPciIoProtocolGuid,\r
- (VOID **) &PciIo,\r
+ (VOID **)&PciIo,\r
This->DriverBindingHandle,\r
Controller,\r
EFI_OPEN_PROTOCOL_BY_DRIVER\r
Private = AllocateZeroPool (sizeof (NVME_CONTROLLER_PRIVATE_DATA));\r
\r
if (Private == NULL) {\r
- DEBUG ((EFI_D_ERROR, "NvmExpressDriverBindingStart: allocating pool for Nvme Private Data failed!\n"));\r
+ DEBUG ((DEBUG_ERROR, "NvmExpressDriverBindingStart: allocating pool for Nvme Private Data failed!\n"));\r
Status = EFI_OUT_OF_RESOURCES;\r
goto Exit;\r
}\r
\r
//\r
- // 4 x 4kB aligned buffers will be carved out of this buffer.\r
+ // Save original PCI attributes\r
+ //\r
+ Status = PciIo->Attributes (\r
+ PciIo,\r
+ EfiPciIoAttributeOperationGet,\r
+ 0,\r
+ &Private->PciAttributes\r
+ );\r
+\r
+ if (EFI_ERROR (Status)) {\r
+ return Status;\r
+ }\r
+\r
+ //\r
+ // Enable 64-bit DMA support in the PCI layer.\r
+ //\r
+ Status = PciIo->Attributes (\r
+ PciIo,\r
+ EfiPciIoAttributeOperationEnable,\r
+ EFI_PCI_IO_ATTRIBUTE_DUAL_ADDRESS_CYCLE,\r
+ NULL\r
+ );\r
+ if (EFI_ERROR (Status)) {\r
+ DEBUG ((DEBUG_WARN, "NvmExpressDriverBindingStart: failed to enable 64-bit DMA (%r)\n", Status));\r
+ }\r
+\r
+ //\r
+ // 6 x 4kB aligned buffers will be carved out of this buffer.\r
// 1st 4kB boundary is the start of the admin submission queue.\r
// 2nd 4kB boundary is the start of the admin completion queue.\r
// 3rd 4kB boundary is the start of I/O submission queue #1.\r
// 4th 4kB boundary is the start of I/O completion queue #1.\r
+ // 5th 4kB boundary is the start of I/O submission queue #2.\r
+ // 6th 4kB boundary is the start of I/O completion queue #2.\r
//\r
- // Allocate 4 pages of memory, then map it for bus master read and write.\r
+ // Allocate 6 pages of memory, then map it for bus master read and write.\r
//\r
Status = PciIo->AllocateBuffer (\r
PciIo,\r
AllocateAnyPages,\r
EfiBootServicesData,\r
- 4,\r
- (VOID**)&Private->Buffer,\r
+ 6,\r
+ (VOID **)&Private->Buffer,\r
0\r
);\r
if (EFI_ERROR (Status)) {\r
goto Exit;\r
}\r
\r
- Bytes = EFI_PAGES_TO_SIZE (4);\r
+ Bytes = EFI_PAGES_TO_SIZE (6);\r
Status = PciIo->Map (\r
PciIo,\r
EfiPciIoOperationBusMasterCommonBuffer,\r
&Private->Mapping\r
);\r
\r
- if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (4))) {\r
+ if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (6))) {\r
goto Exit;\r
}\r
\r
Private->BufferPciAddr = (UINT8 *)(UINTN)MappedAddr;\r
\r
- Private->Signature = NVME_CONTROLLER_PRIVATE_DATA_SIGNATURE;\r
+ Private->Signature = NVME_CONTROLLER_PRIVATE_DATA_SIGNATURE;\r
Private->ControllerHandle = Controller;\r
Private->ImageHandle = This->DriverBindingHandle;\r
Private->DriverBindingHandle = This->DriverBindingHandle;\r
Private->Passthru.BuildDevicePath = NvmExpressBuildDevicePath;\r
Private->Passthru.GetNamespace = NvmExpressGetNamespace;\r
CopyMem (&Private->PassThruMode, &gEfiNvmExpressPassThruMode, sizeof (EFI_NVM_EXPRESS_PASS_THRU_MODE));\r
+ InitializeListHead (&Private->AsyncPassThruQueue);\r
+ InitializeListHead (&Private->UnsubmittedSubtasks);\r
\r
Status = NvmeControllerInit (Private);\r
- if (EFI_ERROR(Status)) {\r
+ if (EFI_ERROR (Status)) {\r
+ goto Exit;\r
+ }\r
+\r
+ //\r
+ // Start the asynchronous I/O completion monitor\r
+ //\r
+ Status = gBS->CreateEvent (\r
+ EVT_TIMER | EVT_NOTIFY_SIGNAL,\r
+ TPL_NOTIFY,\r
+ ProcessAsyncTaskList,\r
+ Private,\r
+ &Private->TimerEvent\r
+ );\r
+ if (EFI_ERROR (Status)) {\r
+ goto Exit;\r
+ }\r
+\r
+ Status = gBS->SetTimer (\r
+ Private->TimerEvent,\r
+ TimerPeriodic,\r
+ NVME_HC_ASYNC_TIMER\r
+ );\r
+ if (EFI_ERROR (Status)) {\r
goto Exit;\r
}\r
\r
if (EFI_ERROR (Status)) {\r
goto Exit;\r
}\r
+\r
+ NvmeRegisterShutdownNotification ();\r
} else {\r
Status = gBS->OpenProtocol (\r
Controller,\r
&gEfiNvmExpressPassThruProtocolGuid,\r
- (VOID **) &Passthru,\r
+ (VOID **)&Passthru,\r
This->DriverBindingHandle,\r
Controller,\r
EFI_OPEN_PROTOCOL_GET_PROTOCOL\r
Status = DiscoverAllNamespaces (\r
Private\r
);\r
-\r
} else if (!IsDevicePathEnd (RemainingDevicePath)) {\r
//\r
// Enumerate the specified NVME namespace\r
}\r
}\r
\r
- DEBUG ((EFI_D_INFO, "NvmExpressDriverBindingStart: end successfully\n"));\r
+ DEBUG ((DEBUG_INFO, "NvmExpressDriverBindingStart: end successfully\n"));\r
return EFI_SUCCESS;\r
\r
Exit:\r
}\r
\r
if ((Private != NULL) && (Private->Buffer != NULL)) {\r
- PciIo->FreeBuffer (PciIo, 4, Private->Buffer);\r
+ PciIo->FreeBuffer (PciIo, 6, Private->Buffer);\r
}\r
\r
if ((Private != NULL) && (Private->ControllerData != NULL)) {\r
}\r
\r
if (Private != NULL) {\r
+ if (Private->TimerEvent != NULL) {\r
+ gBS->CloseEvent (Private->TimerEvent);\r
+ }\r
+\r
FreePool (Private);\r
}\r
\r
Controller\r
);\r
\r
- DEBUG ((EFI_D_INFO, "NvmExpressDriverBindingStart: end with %r\n", Status));\r
+ DEBUG ((DEBUG_INFO, "NvmExpressDriverBindingStart: end with %r\n", Status));\r
\r
return Status;\r
}\r
\r
-\r
/**\r
Stops a device controller or a bus controller.\r
\r
EFI_STATUS\r
EFIAPI\r
NvmExpressDriverBindingStop (\r
- IN EFI_DRIVER_BINDING_PROTOCOL *This,\r
- IN EFI_HANDLE Controller,\r
- IN UINTN NumberOfChildren,\r
- IN EFI_HANDLE *ChildHandleBuffer\r
+ IN EFI_DRIVER_BINDING_PROTOCOL *This,\r
+ IN EFI_HANDLE Controller,\r
+ IN UINTN NumberOfChildren,\r
+ IN EFI_HANDLE *ChildHandleBuffer\r
)\r
{\r
EFI_STATUS Status;\r
UINTN Index;\r
NVME_CONTROLLER_PRIVATE_DATA *Private;\r
EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *PassThru;\r
+ BOOLEAN IsEmpty;\r
+ EFI_TPL OldTpl;\r
\r
if (NumberOfChildren == 0) {\r
Status = gBS->OpenProtocol (\r
Controller,\r
&gEfiNvmExpressPassThruProtocolGuid,\r
- (VOID **) &PassThru,\r
+ (VOID **)&PassThru,\r
This->DriverBindingHandle,\r
Controller,\r
EFI_OPEN_PROTOCOL_GET_PROTOCOL\r
\r
if (!EFI_ERROR (Status)) {\r
Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (PassThru);\r
+\r
+ //\r
+ // Wait for the asynchronous PassThru queue to become empty.\r
+ //\r
+ while (TRUE) {\r
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);\r
+ IsEmpty = IsListEmpty (&Private->AsyncPassThruQueue) &&\r
+ IsListEmpty (&Private->UnsubmittedSubtasks);\r
+ gBS->RestoreTPL (OldTpl);\r
+\r
+ if (IsEmpty) {\r
+ break;\r
+ }\r
+\r
+ gBS->Stall (100);\r
+ }\r
+\r
gBS->UninstallMultipleProtocolInterfaces (\r
- Controller,\r
- &gEfiNvmExpressPassThruProtocolGuid,\r
- PassThru,\r
- NULL\r
- );\r
+ Controller,\r
+ &gEfiNvmExpressPassThruProtocolGuid,\r
+ PassThru,\r
+ NULL\r
+ );\r
+\r
+ if (Private->TimerEvent != NULL) {\r
+ gBS->CloseEvent (Private->TimerEvent);\r
+ }\r
\r
if (Private->Mapping != NULL) {\r
Private->PciIo->Unmap (Private->PciIo, Private->Mapping);\r
}\r
\r
if (Private->Buffer != NULL) {\r
- Private->PciIo->FreeBuffer (Private->PciIo, 4, Private->Buffer);\r
+ Private->PciIo->FreeBuffer (Private->PciIo, 6, Private->Buffer);\r
}\r
\r
FreePool (Private->ControllerData);\r
}\r
\r
gBS->CloseProtocol (\r
- Controller,\r
- &gEfiPciIoProtocolGuid,\r
- This->DriverBindingHandle,\r
- Controller\r
- );\r
+ Controller,\r
+ &gEfiPciIoProtocolGuid,\r
+ This->DriverBindingHandle,\r
+ Controller\r
+ );\r
gBS->CloseProtocol (\r
- Controller,\r
- &gEfiDevicePathProtocolGuid,\r
- This->DriverBindingHandle,\r
- Controller\r
- );\r
+ Controller,\r
+ &gEfiDevicePathProtocolGuid,\r
+ This->DriverBindingHandle,\r
+ Controller\r
+ );\r
+\r
+ NvmeUnregisterShutdownNotification ();\r
+\r
return EFI_SUCCESS;\r
}\r
\r
EFI_STATUS\r
EFIAPI\r
NvmExpressUnload (\r
- IN EFI_HANDLE ImageHandle\r
+ IN EFI_HANDLE ImageHandle\r
)\r
{\r
- EFI_STATUS Status;\r
- EFI_HANDLE *DeviceHandleBuffer;\r
- UINTN DeviceHandleCount;\r
- UINTN Index;\r
- EFI_COMPONENT_NAME_PROTOCOL *ComponentName;\r
- EFI_COMPONENT_NAME2_PROTOCOL *ComponentName2;\r
+ EFI_STATUS Status;\r
+ EFI_HANDLE *DeviceHandleBuffer;\r
+ UINTN DeviceHandleCount;\r
+ UINTN Index;\r
+ EFI_COMPONENT_NAME_PROTOCOL *ComponentName;\r
+ EFI_COMPONENT_NAME2_PROTOCOL *ComponentName2;\r
\r
//\r
// Get the list of the device handles managed by this driver.\r
// those protocols installed at image handle.\r
//\r
DeviceHandleBuffer = NULL;\r
- Status = gBS->LocateHandleBuffer (\r
- ByProtocol,\r
- &gEfiNvmExpressPassThruProtocolGuid,\r
- NULL,\r
- &DeviceHandleCount,\r
- &DeviceHandleBuffer\r
- );\r
+ Status = gBS->LocateHandleBuffer (\r
+ ByProtocol,\r
+ &gEfiNvmExpressPassThruProtocolGuid,\r
+ NULL,\r
+ &DeviceHandleCount,\r
+ &DeviceHandleBuffer\r
+ );\r
\r
if (!EFI_ERROR (Status)) {\r
//\r
Status = gBS->HandleProtocol (\r
ImageHandle,\r
&gEfiComponentNameProtocolGuid,\r
- (VOID **) &ComponentName\r
+ (VOID **)&ComponentName\r
);\r
if (!EFI_ERROR (Status)) {\r
gBS->UninstallProtocolInterface (\r
Status = gBS->HandleProtocol (\r
ImageHandle,\r
&gEfiComponentName2ProtocolGuid,\r
- (VOID **) &ComponentName2\r
+ (VOID **)&ComponentName2\r
);\r
if (!EFI_ERROR (Status)) {\r
gBS->UninstallProtocolInterface (\r
if (DeviceHandleBuffer != NULL) {\r
gBS->FreePool (DeviceHandleBuffer);\r
}\r
+\r
return Status;\r
}\r
\r
IN EFI_SYSTEM_TABLE *SystemTable\r
)\r
{\r
- EFI_STATUS Status;\r
+ EFI_STATUS Status;\r
\r
Status = EfiLibInstallDriverBindingComponentName2 (\r
ImageHandle,\r
// EFI drivers that are on PCI and other plug in cards.\r
//\r
gNvmExpressDriverSupportedEfiVersion.FirmwareVersion = 0x00020028;\r
- Status = gBS->InstallMultipleProtocolInterfaces (\r
- &ImageHandle,\r
- &gEfiDriverSupportedEfiVersionProtocolGuid,\r
- &gNvmExpressDriverSupportedEfiVersion,\r
- NULL\r
- );\r
+ Status = gBS->InstallMultipleProtocolInterfaces (\r
+ &ImageHandle,\r
+ &gEfiDriverSupportedEfiVersionProtocolGuid,\r
+ &gNvmExpressDriverSupportedEfiVersion,\r
+ NULL\r
+ );\r
ASSERT_EFI_ERROR (Status);\r
return Status;\r
}\r
projects / mirror_edk2.git / blobdiff