NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows\r
NVM Express specification.\r
\r
- Copyright (c) 2013 - 2016, Intel Corporation. All rights reserved.<BR>\r
+ Copyright (c) 2013 - 2017, Intel Corporation. All rights reserved.<BR>\r
This program and the accompanying materials\r
are licensed and made available under the terms and conditions of the BSD License\r
which accompanies this distribution. The full text of the license may be found at\r
// Template for NVM Express Pass Thru Mode data structure.\r
//\r
GLOBAL_REMOVE_IF_UNREFERENCED EFI_NVM_EXPRESS_PASS_THRU_MODE gEfiNvmExpressPassThruMode = {\r
- EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL | EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL | EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_CMD_SET_NVM,\r
+ EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL |\r
+ EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL |\r
+ EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_NONBLOCKIO |\r
+ EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_CMD_SET_NVM,\r
sizeof (UINTN),\r
0x10100\r
};\r
UINT32 Lbads;\r
UINT32 Flbas;\r
UINT32 LbaFmtIdx;\r
+ UINT8 Sn[21];\r
+ UINT8 Mn[41];\r
+ VOID *DummyInterface;\r
\r
NewDevicePathNode = NULL;\r
DevicePath = NULL;\r
Device->Media.LogicalPartition = FALSE;\r
Device->Media.ReadOnly = FALSE;\r
Device->Media.WriteCaching = FALSE;\r
+ Device->Media.IoAlign = Private->PassThruMode.IoAlign;\r
\r
Flbas = NamespaceData->Flbas;\r
LbaFmtIdx = Flbas & 0xF;\r
Device->BlockIo.WriteBlocks = NvmeBlockIoWriteBlocks;\r
Device->BlockIo.FlushBlocks = NvmeBlockIoFlushBlocks;\r
\r
+ //\r
+ // Create BlockIo2 Protocol instance\r
+ //\r
+ Device->BlockIo2.Media = &Device->Media;\r
+ Device->BlockIo2.Reset = NvmeBlockIoResetEx;\r
+ Device->BlockIo2.ReadBlocksEx = NvmeBlockIoReadBlocksEx;\r
+ Device->BlockIo2.WriteBlocksEx = NvmeBlockIoWriteBlocksEx;\r
+ Device->BlockIo2.FlushBlocksEx = NvmeBlockIoFlushBlocksEx;\r
+ InitializeListHead (&Device->AsyncQueue);\r
+\r
//\r
// Create StorageSecurityProtocol Instance\r
//\r
Device->DevicePath,\r
&gEfiBlockIoProtocolGuid,\r
&Device->BlockIo,\r
+ &gEfiBlockIo2ProtocolGuid,\r
+ &Device->BlockIo2,\r
&gEfiDiskInfoProtocolGuid,\r
&Device->DiskInfo,\r
NULL\r
Device->DevicePath,\r
&gEfiBlockIoProtocolGuid,\r
&Device->BlockIo,\r
+ &gEfiBlockIo2ProtocolGuid,\r
+ &Device->BlockIo2,\r
&gEfiDiskInfoProtocolGuid,\r
&Device->DiskInfo,\r
NULL\r
gBS->OpenProtocol (\r
Private->ControllerHandle,\r
&gEfiNvmExpressPassThruProtocolGuid,\r
- (VOID **) &Private->Passthru,\r
+ (VOID **) &DummyInterface,\r
Private->DriverBindingHandle,\r
Device->DeviceHandle,\r
EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER\r
//\r
// Build controller name for Component Name (2) protocol.\r
//\r
- UnicodeSPrintAsciiFormat (Device->ModelName, sizeof (Device->ModelName), "%a-%a-%x", Private->ControllerData->Sn, Private->ControllerData->Mn, NamespaceData->Eui64);\r
+ CopyMem (Sn, Private->ControllerData->Sn, sizeof (Private->ControllerData->Sn));\r
+ Sn[20] = 0;\r
+ CopyMem (Mn, Private->ControllerData->Mn, sizeof (Private->ControllerData->Mn));\r
+ Mn[40] = 0;\r
+ UnicodeSPrintAsciiFormat (Device->ModelName, sizeof (Device->ModelName), "%a-%a-%x", Sn, Mn, NamespaceData->Eui64);\r
\r
AddUnicodeString2 (\r
"eng",\r
EFI_STATUS Status;\r
EFI_BLOCK_IO_PROTOCOL *BlockIo;\r
NVME_DEVICE_PRIVATE_DATA *Device;\r
- NVME_CONTROLLER_PRIVATE_DATA *Private;\r
EFI_STORAGE_SECURITY_COMMAND_PROTOCOL *StorageSecurity;\r
+ BOOLEAN IsEmpty;\r
+ EFI_TPL OldTpl;\r
+ VOID *DummyInterface;\r
\r
BlockIo = NULL;\r
\r
}\r
\r
Device = NVME_DEVICE_PRIVATE_DATA_FROM_BLOCK_IO (BlockIo);\r
- Private = Device->Controller;\r
+\r
+ //\r
+ // Wait for the device's asynchronous I/O queue to become empty.\r
+ //\r
+ while (TRUE) {\r
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);\r
+ IsEmpty = IsListEmpty (&Device->AsyncQueue);\r
+ gBS->RestoreTPL (OldTpl);\r
+\r
+ if (IsEmpty) {\r
+ break;\r
+ }\r
+\r
+ gBS->Stall (100);\r
+ }\r
\r
//\r
// Close the child handle\r
Device->DevicePath,\r
&gEfiBlockIoProtocolGuid,\r
&Device->BlockIo,\r
+ &gEfiBlockIo2ProtocolGuid,\r
+ &Device->BlockIo2,\r
&gEfiDiskInfoProtocolGuid,\r
&Device->DiskInfo,\r
NULL\r
gBS->OpenProtocol (\r
Controller,\r
&gEfiNvmExpressPassThruProtocolGuid,\r
- (VOID **) &Private->Passthru,\r
+ (VOID **) &DummyInterface,\r
This->DriverBindingHandle,\r
Handle,\r
EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER\r
gBS->OpenProtocol (\r
Controller,\r
&gEfiNvmExpressPassThruProtocolGuid,\r
- (VOID **) &Private->Passthru,\r
+ (VOID **) &DummyInterface,\r
This->DriverBindingHandle,\r
Handle,\r
EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER\r
return EFI_SUCCESS;\r
}\r
\r
+/**\r
+ Call back function when the timer event is signaled.\r
+\r
+ @param[in] Event The Event this notify function registered to.\r
+ @param[in] Context Pointer to the context data registered to the\r
+ Event.\r
+\r
+**/\r
+VOID\r
+EFIAPI\r
+ProcessAsyncTaskList (\r
+ IN EFI_EVENT Event,\r
+ IN VOID* Context\r
+ )\r
+{\r
+ NVME_CONTROLLER_PRIVATE_DATA *Private;\r
+ EFI_PCI_IO_PROTOCOL *PciIo;\r
+ NVME_CQ *Cq;\r
+ UINT16 QueueId;\r
+ UINT32 Data;\r
+ LIST_ENTRY *Link;\r
+ LIST_ENTRY *NextLink;\r
+ NVME_PASS_THRU_ASYNC_REQ *AsyncRequest;\r
+ NVME_BLKIO2_SUBTASK *Subtask;\r
+ NVME_BLKIO2_REQUEST *BlkIo2Request;\r
+ EFI_BLOCK_IO2_TOKEN *Token;\r
+ BOOLEAN HasNewItem;\r
+ EFI_STATUS Status;\r
+\r
+ Private = (NVME_CONTROLLER_PRIVATE_DATA*)Context;\r
+ QueueId = 2;\r
+ Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;\r
+ HasNewItem = FALSE;\r
+ PciIo = Private->PciIo;\r
+\r
+ //\r
+ // Submit asynchronous subtasks to the NVMe Submission Queue\r
+ //\r
+ for (Link = GetFirstNode (&Private->UnsubmittedSubtasks);\r
+ !IsNull (&Private->UnsubmittedSubtasks, Link);\r
+ Link = NextLink) {\r
+ NextLink = GetNextNode (&Private->UnsubmittedSubtasks, Link);\r
+ Subtask = NVME_BLKIO2_SUBTASK_FROM_LINK (Link);\r
+ BlkIo2Request = Subtask->BlockIo2Request;\r
+ Token = BlkIo2Request->Token;\r
+ RemoveEntryList (Link);\r
+ BlkIo2Request->UnsubmittedSubtaskNum--;\r
+\r
+ //\r
+ // If any previous subtask fails, do not process subsequent ones.\r
+ //\r
+ if (Token->TransactionStatus != EFI_SUCCESS) {\r
+ if (IsListEmpty (&BlkIo2Request->SubtasksQueue) &&\r
+ BlkIo2Request->LastSubtaskSubmitted &&\r
+ (BlkIo2Request->UnsubmittedSubtaskNum == 0)) {\r
+ //\r
+ // Remove the BlockIo2 request from the device asynchronous queue.\r
+ //\r
+ RemoveEntryList (&BlkIo2Request->Link);\r
+ FreePool (BlkIo2Request);\r
+ gBS->SignalEvent (Token->Event);\r
+ }\r
+\r
+ FreePool (Subtask->CommandPacket->NvmeCmd);\r
+ FreePool (Subtask->CommandPacket->NvmeCompletion);\r
+ FreePool (Subtask->CommandPacket);\r
+ FreePool (Subtask);\r
+\r
+ continue;\r
+ }\r
+\r
+ Status = Private->Passthru.PassThru (\r
+ &Private->Passthru,\r
+ Subtask->NamespaceId,\r
+ Subtask->CommandPacket,\r
+ Subtask->Event\r
+ );\r
+ if (Status == EFI_NOT_READY) {\r
+ InsertHeadList (&Private->UnsubmittedSubtasks, Link);\r
+ BlkIo2Request->UnsubmittedSubtaskNum++;\r
+ break;\r
+ } else if (EFI_ERROR (Status)) {\r
+ Token->TransactionStatus = EFI_DEVICE_ERROR;\r
+\r
+ if (IsListEmpty (&BlkIo2Request->SubtasksQueue) &&\r
+ Subtask->IsLast) {\r
+ //\r
+ // Remove the BlockIo2 request from the device asynchronous queue.\r
+ //\r
+ RemoveEntryList (&BlkIo2Request->Link);\r
+ FreePool (BlkIo2Request);\r
+ gBS->SignalEvent (Token->Event);\r
+ }\r
+\r
+ FreePool (Subtask->CommandPacket->NvmeCmd);\r
+ FreePool (Subtask->CommandPacket->NvmeCompletion);\r
+ FreePool (Subtask->CommandPacket);\r
+ FreePool (Subtask);\r
+ } else {\r
+ InsertTailList (&BlkIo2Request->SubtasksQueue, Link);\r
+ if (Subtask->IsLast) {\r
+ BlkIo2Request->LastSubtaskSubmitted = TRUE;\r
+ }\r
+ }\r
+ }\r
+\r
+ while (Cq->Pt != Private->Pt[QueueId]) {\r
+ ASSERT (Cq->Sqid == QueueId);\r
+\r
+ HasNewItem = TRUE;\r
+\r
+ //\r
+ // Find the command with given Command Id.\r
+ //\r
+ for (Link = GetFirstNode (&Private->AsyncPassThruQueue);\r
+ !IsNull (&Private->AsyncPassThruQueue, Link);\r
+ Link = NextLink) {\r
+ NextLink = GetNextNode (&Private->AsyncPassThruQueue, Link);\r
+ AsyncRequest = NVME_PASS_THRU_ASYNC_REQ_FROM_THIS (Link);\r
+ if (AsyncRequest->CommandId == Cq->Cid) {\r
+ //\r
+ // Copy the Respose Queue entry for this command to the callers\r
+ // response buffer.\r
+ //\r
+ CopyMem (\r
+ AsyncRequest->Packet->NvmeCompletion,\r
+ Cq,\r
+ sizeof(EFI_NVM_EXPRESS_COMPLETION)\r
+ );\r
+\r
+ //\r
+ // Free the resources allocated before cmd submission\r
+ //\r
+ if (AsyncRequest->MapData != NULL) {\r
+ PciIo->Unmap (PciIo, AsyncRequest->MapData);\r
+ }\r
+ if (AsyncRequest->MapMeta != NULL) {\r
+ PciIo->Unmap (PciIo, AsyncRequest->MapMeta);\r
+ }\r
+ if (AsyncRequest->MapPrpList != NULL) {\r
+ PciIo->Unmap (PciIo, AsyncRequest->MapPrpList);\r
+ }\r
+ if (AsyncRequest->PrpListHost != NULL) {\r
+ PciIo->FreeBuffer (\r
+ PciIo,\r
+ AsyncRequest->PrpListNo,\r
+ AsyncRequest->PrpListHost\r
+ );\r
+ }\r
+\r
+ RemoveEntryList (Link);\r
+ gBS->SignalEvent (AsyncRequest->CallerEvent);\r
+ FreePool (AsyncRequest);\r
+\r
+ //\r
+ // Update submission queue head.\r
+ //\r
+ Private->AsyncSqHead = Cq->Sqhd;\r
+ break;\r
+ }\r
+ }\r
+\r
+ Private->CqHdbl[QueueId].Cqh++;\r
+ if (Private->CqHdbl[QueueId].Cqh > NVME_ASYNC_CCQ_SIZE) {\r
+ Private->CqHdbl[QueueId].Cqh = 0;\r
+ Private->Pt[QueueId] ^= 1;\r
+ }\r
+\r
+ Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;\r
+ }\r
+\r
+ if (HasNewItem) {\r
+ Data = ReadUnaligned32 ((UINT32*)&Private->CqHdbl[QueueId]);\r
+ PciIo->Mem.Write (\r
+ PciIo,\r
+ EfiPciIoWidthUint32,\r
+ NVME_BAR,\r
+ NVME_CQHDBL_OFFSET(QueueId, Private->Cap.Dstrd),\r
+ 1,\r
+ &Data\r
+ );\r
+ }\r
+}\r
+\r
/**\r
Tests to see if this driver supports a given controller. If a child device is provided,\r
it further tests to see if this driver supports creating a handle for the specified child device.\r
}\r
\r
//\r
- // 4 x 4kB aligned buffers will be carved out of this buffer.\r
+ // 6 x 4kB aligned buffers will be carved out of this buffer.\r
// 1st 4kB boundary is the start of the admin submission queue.\r
// 2nd 4kB boundary is the start of the admin completion queue.\r
// 3rd 4kB boundary is the start of I/O submission queue #1.\r
// 4th 4kB boundary is the start of I/O completion queue #1.\r
+ // 5th 4kB boundary is the start of I/O submission queue #2.\r
+ // 6th 4kB boundary is the start of I/O completion queue #2.\r
//\r
- // Allocate 4 pages of memory, then map it for bus master read and write.\r
+ // Allocate 6 pages of memory, then map it for bus master read and write.\r
//\r
Status = PciIo->AllocateBuffer (\r
PciIo,\r
AllocateAnyPages,\r
EfiBootServicesData,\r
- 4,\r
+ 6,\r
(VOID**)&Private->Buffer,\r
0\r
);\r
goto Exit;\r
}\r
\r
- Bytes = EFI_PAGES_TO_SIZE (4);\r
+ Bytes = EFI_PAGES_TO_SIZE (6);\r
Status = PciIo->Map (\r
PciIo,\r
EfiPciIoOperationBusMasterCommonBuffer,\r
&Private->Mapping\r
);\r
\r
- if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (4))) {\r
+ if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (6))) {\r
goto Exit;\r
}\r
\r
Private->BufferPciAddr = (UINT8 *)(UINTN)MappedAddr;\r
- ZeroMem (Private->Buffer, EFI_PAGES_TO_SIZE (4));\r
\r
Private->Signature = NVME_CONTROLLER_PRIVATE_DATA_SIGNATURE;\r
Private->ControllerHandle = Controller;\r
Private->Passthru.BuildDevicePath = NvmExpressBuildDevicePath;\r
Private->Passthru.GetNamespace = NvmExpressGetNamespace;\r
CopyMem (&Private->PassThruMode, &gEfiNvmExpressPassThruMode, sizeof (EFI_NVM_EXPRESS_PASS_THRU_MODE));\r
+ InitializeListHead (&Private->AsyncPassThruQueue);\r
+ InitializeListHead (&Private->UnsubmittedSubtasks);\r
\r
Status = NvmeControllerInit (Private);\r
if (EFI_ERROR(Status)) {\r
goto Exit;\r
}\r
\r
+ //\r
+ // Start the asynchronous I/O completion monitor\r
+ //\r
+ Status = gBS->CreateEvent (\r
+ EVT_TIMER | EVT_NOTIFY_SIGNAL,\r
+ TPL_NOTIFY,\r
+ ProcessAsyncTaskList,\r
+ Private,\r
+ &Private->TimerEvent\r
+ );\r
+ if (EFI_ERROR (Status)) {\r
+ goto Exit;\r
+ }\r
+\r
+ Status = gBS->SetTimer (\r
+ Private->TimerEvent,\r
+ TimerPeriodic,\r
+ NVME_HC_ASYNC_TIMER\r
+ );\r
+ if (EFI_ERROR (Status)) {\r
+ goto Exit;\r
+ }\r
+\r
Status = gBS->InstallMultipleProtocolInterfaces (\r
&Controller,\r
&gEfiNvmExpressPassThruProtocolGuid,\r
if (EFI_ERROR (Status)) {\r
goto Exit;\r
}\r
+\r
+ NvmeRegisterShutdownNotification ();\r
} else {\r
Status = gBS->OpenProtocol (\r
Controller,\r
}\r
\r
if ((Private != NULL) && (Private->Buffer != NULL)) {\r
- PciIo->FreeBuffer (PciIo, 4, Private->Buffer);\r
+ PciIo->FreeBuffer (PciIo, 6, Private->Buffer);\r
}\r
\r
if ((Private != NULL) && (Private->ControllerData != NULL)) {\r
}\r
\r
if (Private != NULL) {\r
+ if (Private->TimerEvent != NULL) {\r
+ gBS->CloseEvent (Private->TimerEvent);\r
+ }\r
+\r
FreePool (Private);\r
}\r
\r
UINTN Index;\r
NVME_CONTROLLER_PRIVATE_DATA *Private;\r
EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *PassThru;\r
+ BOOLEAN IsEmpty;\r
+ EFI_TPL OldTpl;\r
\r
if (NumberOfChildren == 0) {\r
Status = gBS->OpenProtocol (\r
\r
if (!EFI_ERROR (Status)) {\r
Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (PassThru);\r
+\r
+ //\r
+ // Wait for the asynchronous PassThru queue to become empty.\r
+ //\r
+ while (TRUE) {\r
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);\r
+ IsEmpty = IsListEmpty (&Private->AsyncPassThruQueue) &&\r
+ IsListEmpty (&Private->UnsubmittedSubtasks);\r
+ gBS->RestoreTPL (OldTpl);\r
+\r
+ if (IsEmpty) {\r
+ break;\r
+ }\r
+\r
+ gBS->Stall (100);\r
+ }\r
+\r
gBS->UninstallMultipleProtocolInterfaces (\r
Controller,\r
&gEfiNvmExpressPassThruProtocolGuid,\r
NULL\r
);\r
\r
+ if (Private->TimerEvent != NULL) {\r
+ gBS->CloseEvent (Private->TimerEvent);\r
+ }\r
+\r
if (Private->Mapping != NULL) {\r
Private->PciIo->Unmap (Private->PciIo, Private->Mapping);\r
}\r
\r
if (Private->Buffer != NULL) {\r
- Private->PciIo->FreeBuffer (Private->PciIo, 4, Private->Buffer);\r
+ Private->PciIo->FreeBuffer (Private->PciIo, 6, Private->Buffer);\r
}\r
\r
FreePool (Private->ControllerData);\r
This->DriverBindingHandle,\r
Controller\r
);\r
+\r
+ NvmeUnregisterShutdownNotification ();\r
+\r
return EFI_SUCCESS;\r
}\r
\r
projects / mirror_edk2.git / blobdiff