NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows\r
NVM Express specification.\r
\r
- Copyright (c) 2013 - 2016, Intel Corporation. All rights reserved.<BR>\r
- This program and the accompanying materials\r
- are licensed and made available under the terms and conditions of the BSD License\r
- which accompanies this distribution. The full text of the license may be found at\r
- http://opensource.org/licenses/bsd-license.php.\r
-\r
- THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,\r
- WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.\r
+ Copyright (c) 2013 - 2017, Intel Corporation. All rights reserved.<BR>\r
+ SPDX-License-Identifier: BSD-2-Clause-Patent\r
\r
**/\r
\r
// Template for NVM Express Pass Thru Mode data structure.\r
//\r
GLOBAL_REMOVE_IF_UNREFERENCED EFI_NVM_EXPRESS_PASS_THRU_MODE gEfiNvmExpressPassThruMode = {\r
- EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL | EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL | EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_CMD_SET_NVM,\r
+ EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL |\r
+ EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL |\r
+ EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_NONBLOCKIO |\r
+ EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_CMD_SET_NVM,\r
sizeof (UINTN),\r
0x10100\r
};\r
UINT32 Lbads;\r
UINT32 Flbas;\r
UINT32 LbaFmtIdx;\r
+ UINT8 Sn[21];\r
+ UINT8 Mn[41];\r
+ VOID *DummyInterface;\r
\r
NewDevicePathNode = NULL;\r
DevicePath = NULL;\r
Device->Media.LogicalPartition = FALSE;\r
Device->Media.ReadOnly = FALSE;\r
Device->Media.WriteCaching = FALSE;\r
+ Device->Media.IoAlign = Private->PassThruMode.IoAlign;\r
\r
Flbas = NamespaceData->Flbas;\r
LbaFmtIdx = Flbas & 0xF;\r
Device->BlockIo.WriteBlocks = NvmeBlockIoWriteBlocks;\r
Device->BlockIo.FlushBlocks = NvmeBlockIoFlushBlocks;\r
\r
+ //\r
+ // Create BlockIo2 Protocol instance\r
+ //\r
+ Device->BlockIo2.Media = &Device->Media;\r
+ Device->BlockIo2.Reset = NvmeBlockIoResetEx;\r
+ Device->BlockIo2.ReadBlocksEx = NvmeBlockIoReadBlocksEx;\r
+ Device->BlockIo2.WriteBlocksEx = NvmeBlockIoWriteBlocksEx;\r
+ Device->BlockIo2.FlushBlocksEx = NvmeBlockIoFlushBlocksEx;\r
+ InitializeListHead (&Device->AsyncQueue);\r
+\r
//\r
// Create StorageSecurityProtocol Instance\r
//\r
Device->DevicePath,\r
&gEfiBlockIoProtocolGuid,\r
&Device->BlockIo,\r
+ &gEfiBlockIo2ProtocolGuid,\r
+ &Device->BlockIo2,\r
&gEfiDiskInfoProtocolGuid,\r
&Device->DiskInfo,\r
NULL\r
Device->DevicePath,\r
&gEfiBlockIoProtocolGuid,\r
&Device->BlockIo,\r
+ &gEfiBlockIo2ProtocolGuid,\r
+ &Device->BlockIo2,\r
&gEfiDiskInfoProtocolGuid,\r
&Device->DiskInfo,\r
NULL\r
gBS->OpenProtocol (\r
Private->ControllerHandle,\r
&gEfiNvmExpressPassThruProtocolGuid,\r
- (VOID **) &Private->Passthru,\r
+ (VOID **) &DummyInterface,\r
Private->DriverBindingHandle,\r
Device->DeviceHandle,\r
EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER\r
//\r
// Build controller name for Component Name (2) protocol.\r
//\r
- UnicodeSPrintAsciiFormat (Device->ModelName, sizeof (Device->ModelName), "%a-%a-%x", Private->ControllerData->Sn, Private->ControllerData->Mn, NamespaceData->Eui64);\r
+ CopyMem (Sn, Private->ControllerData->Sn, sizeof (Private->ControllerData->Sn));\r
+ Sn[20] = 0;\r
+ CopyMem (Mn, Private->ControllerData->Mn, sizeof (Private->ControllerData->Mn));\r
+ Mn[40] = 0;\r
+ UnicodeSPrintAsciiFormat (Device->ModelName, sizeof (Device->ModelName), "%a-%a-%x", Sn, Mn, NamespaceData->Eui64);\r
\r
AddUnicodeString2 (\r
"eng",\r
EFI_STATUS Status;\r
EFI_BLOCK_IO_PROTOCOL *BlockIo;\r
NVME_DEVICE_PRIVATE_DATA *Device;\r
- NVME_CONTROLLER_PRIVATE_DATA *Private;\r
EFI_STORAGE_SECURITY_COMMAND_PROTOCOL *StorageSecurity;\r
+ BOOLEAN IsEmpty;\r
+ EFI_TPL OldTpl;\r
+ VOID *DummyInterface;\r
\r
BlockIo = NULL;\r
\r
}\r
\r
Device = NVME_DEVICE_PRIVATE_DATA_FROM_BLOCK_IO (BlockIo);\r
- Private = Device->Controller;\r
+\r
+ //\r
+ // Wait for the device's asynchronous I/O queue to become empty.\r
+ //\r
+ while (TRUE) {\r
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);\r
+ IsEmpty = IsListEmpty (&Device->AsyncQueue);\r
+ gBS->RestoreTPL (OldTpl);\r
+\r
+ if (IsEmpty) {\r
+ break;\r
+ }\r
+\r
+ gBS->Stall (100);\r
+ }\r
\r
//\r
// Close the child handle\r
Device->DevicePath,\r
&gEfiBlockIoProtocolGuid,\r
&Device->BlockIo,\r
+ &gEfiBlockIo2ProtocolGuid,\r
+ &Device->BlockIo2,\r
&gEfiDiskInfoProtocolGuid,\r
&Device->DiskInfo,\r
NULL\r
gBS->OpenProtocol (\r
Controller,\r
&gEfiNvmExpressPassThruProtocolGuid,\r
- (VOID **) &Private->Passthru,\r
+ (VOID **) &DummyInterface,\r
This->DriverBindingHandle,\r
Handle,\r
EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER\r
gBS->OpenProtocol (\r
Controller,\r
&gEfiNvmExpressPassThruProtocolGuid,\r
- (VOID **) &Private->Passthru,\r
+ (VOID **) &DummyInterface,\r
This->DriverBindingHandle,\r
Handle,\r
EFI_OPEN_PROTOCOL_BY_CHILD_CONTROLLER\r
return EFI_SUCCESS;\r
}\r
\r
+/**\r
+ Call back function when the timer event is signaled.\r
+\r
+ @param[in] Event The Event this notify function registered to.\r
+ @param[in] Context Pointer to the context data registered to the\r
+ Event.\r
+\r
+**/\r
+VOID\r
+EFIAPI\r
+ProcessAsyncTaskList (\r
+ IN EFI_EVENT Event,\r
+ IN VOID* Context\r
+ )\r
+{\r
+ NVME_CONTROLLER_PRIVATE_DATA *Private;\r
+ EFI_PCI_IO_PROTOCOL *PciIo;\r
+ NVME_CQ *Cq;\r
+ UINT16 QueueId;\r
+ UINT32 Data;\r
+ LIST_ENTRY *Link;\r
+ LIST_ENTRY *NextLink;\r
+ NVME_PASS_THRU_ASYNC_REQ *AsyncRequest;\r
+ NVME_BLKIO2_SUBTASK *Subtask;\r
+ NVME_BLKIO2_REQUEST *BlkIo2Request;\r
+ EFI_BLOCK_IO2_TOKEN *Token;\r
+ BOOLEAN HasNewItem;\r
+ EFI_STATUS Status;\r
+\r
+ Private = (NVME_CONTROLLER_PRIVATE_DATA*)Context;\r
+ QueueId = 2;\r
+ Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;\r
+ HasNewItem = FALSE;\r
+ PciIo = Private->PciIo;\r
+\r
+ //\r
+ // Submit asynchronous subtasks to the NVMe Submission Queue\r
+ //\r
+ for (Link = GetFirstNode (&Private->UnsubmittedSubtasks);\r
+ !IsNull (&Private->UnsubmittedSubtasks, Link);\r
+ Link = NextLink) {\r
+ NextLink = GetNextNode (&Private->UnsubmittedSubtasks, Link);\r
+ Subtask = NVME_BLKIO2_SUBTASK_FROM_LINK (Link);\r
+ BlkIo2Request = Subtask->BlockIo2Request;\r
+ Token = BlkIo2Request->Token;\r
+ RemoveEntryList (Link);\r
+ BlkIo2Request->UnsubmittedSubtaskNum--;\r
+\r
+ //\r
+ // If any previous subtask fails, do not process subsequent ones.\r
+ //\r
+ if (Token->TransactionStatus != EFI_SUCCESS) {\r
+ if (IsListEmpty (&BlkIo2Request->SubtasksQueue) &&\r
+ BlkIo2Request->LastSubtaskSubmitted &&\r
+ (BlkIo2Request->UnsubmittedSubtaskNum == 0)) {\r
+ //\r
+ // Remove the BlockIo2 request from the device asynchronous queue.\r
+ //\r
+ RemoveEntryList (&BlkIo2Request->Link);\r
+ FreePool (BlkIo2Request);\r
+ gBS->SignalEvent (Token->Event);\r
+ }\r
+\r
+ FreePool (Subtask->CommandPacket->NvmeCmd);\r
+ FreePool (Subtask->CommandPacket->NvmeCompletion);\r
+ FreePool (Subtask->CommandPacket);\r
+ FreePool (Subtask);\r
+\r
+ continue;\r
+ }\r
+\r
+ Status = Private->Passthru.PassThru (\r
+ &Private->Passthru,\r
+ Subtask->NamespaceId,\r
+ Subtask->CommandPacket,\r
+ Subtask->Event\r
+ );\r
+ if (Status == EFI_NOT_READY) {\r
+ InsertHeadList (&Private->UnsubmittedSubtasks, Link);\r
+ BlkIo2Request->UnsubmittedSubtaskNum++;\r
+ break;\r
+ } else if (EFI_ERROR (Status)) {\r
+ Token->TransactionStatus = EFI_DEVICE_ERROR;\r
+\r
+ if (IsListEmpty (&BlkIo2Request->SubtasksQueue) &&\r
+ Subtask->IsLast) {\r
+ //\r
+ // Remove the BlockIo2 request from the device asynchronous queue.\r
+ //\r
+ RemoveEntryList (&BlkIo2Request->Link);\r
+ FreePool (BlkIo2Request);\r
+ gBS->SignalEvent (Token->Event);\r
+ }\r
+\r
+ FreePool (Subtask->CommandPacket->NvmeCmd);\r
+ FreePool (Subtask->CommandPacket->NvmeCompletion);\r
+ FreePool (Subtask->CommandPacket);\r
+ FreePool (Subtask);\r
+ } else {\r
+ InsertTailList (&BlkIo2Request->SubtasksQueue, Link);\r
+ if (Subtask->IsLast) {\r
+ BlkIo2Request->LastSubtaskSubmitted = TRUE;\r
+ }\r
+ }\r
+ }\r
+\r
+ while (Cq->Pt != Private->Pt[QueueId]) {\r
+ ASSERT (Cq->Sqid == QueueId);\r
+\r
+ HasNewItem = TRUE;\r
+\r
+ //\r
+ // Find the command with given Command Id.\r
+ //\r
+ for (Link = GetFirstNode (&Private->AsyncPassThruQueue);\r
+ !IsNull (&Private->AsyncPassThruQueue, Link);\r
+ Link = NextLink) {\r
+ NextLink = GetNextNode (&Private->AsyncPassThruQueue, Link);\r
+ AsyncRequest = NVME_PASS_THRU_ASYNC_REQ_FROM_THIS (Link);\r
+ if (AsyncRequest->CommandId == Cq->Cid) {\r
+ //\r
+ // Copy the Respose Queue entry for this command to the callers\r
+ // response buffer.\r
+ //\r
+ CopyMem (\r
+ AsyncRequest->Packet->NvmeCompletion,\r
+ Cq,\r
+ sizeof(EFI_NVM_EXPRESS_COMPLETION)\r
+ );\r
+\r
+ //\r
+ // Free the resources allocated before cmd submission\r
+ //\r
+ if (AsyncRequest->MapData != NULL) {\r
+ PciIo->Unmap (PciIo, AsyncRequest->MapData);\r
+ }\r
+ if (AsyncRequest->MapMeta != NULL) {\r
+ PciIo->Unmap (PciIo, AsyncRequest->MapMeta);\r
+ }\r
+ if (AsyncRequest->MapPrpList != NULL) {\r
+ PciIo->Unmap (PciIo, AsyncRequest->MapPrpList);\r
+ }\r
+ if (AsyncRequest->PrpListHost != NULL) {\r
+ PciIo->FreeBuffer (\r
+ PciIo,\r
+ AsyncRequest->PrpListNo,\r
+ AsyncRequest->PrpListHost\r
+ );\r
+ }\r
+\r
+ RemoveEntryList (Link);\r
+ gBS->SignalEvent (AsyncRequest->CallerEvent);\r
+ FreePool (AsyncRequest);\r
+\r
+ //\r
+ // Update submission queue head.\r
+ //\r
+ Private->AsyncSqHead = Cq->Sqhd;\r
+ break;\r
+ }\r
+ }\r
+\r
+ Private->CqHdbl[QueueId].Cqh++;\r
+ if (Private->CqHdbl[QueueId].Cqh > NVME_ASYNC_CCQ_SIZE) {\r
+ Private->CqHdbl[QueueId].Cqh = 0;\r
+ Private->Pt[QueueId] ^= 1;\r
+ }\r
+\r
+ Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;\r
+ }\r
+\r
+ if (HasNewItem) {\r
+ Data = ReadUnaligned32 ((UINT32*)&Private->CqHdbl[QueueId]);\r
+ PciIo->Mem.Write (\r
+ PciIo,\r
+ EfiPciIoWidthUint32,\r
+ NVME_BAR,\r
+ NVME_CQHDBL_OFFSET(QueueId, Private->Cap.Dstrd),\r
+ 1,\r
+ &Data\r
+ );\r
+ }\r
+}\r
+\r
/**\r
Tests to see if this driver supports a given controller. If a child device is provided,\r
it further tests to see if this driver supports creating a handle for the specified child device.\r
}\r
\r
//\r
- // 4 x 4kB aligned buffers will be carved out of this buffer.\r
+ // 6 x 4kB aligned buffers will be carved out of this buffer.\r
// 1st 4kB boundary is the start of the admin submission queue.\r
// 2nd 4kB boundary is the start of the admin completion queue.\r
// 3rd 4kB boundary is the start of I/O submission queue #1.\r
// 4th 4kB boundary is the start of I/O completion queue #1.\r
+ // 5th 4kB boundary is the start of I/O submission queue #2.\r
+ // 6th 4kB boundary is the start of I/O completion queue #2.\r
//\r
- // Allocate 4 pages of memory, then map it for bus master read and write.\r
+ // Allocate 6 pages of memory, then map it for bus master read and write.\r
//\r
Status = PciIo->AllocateBuffer (\r
PciIo,\r
AllocateAnyPages,\r
EfiBootServicesData,\r
- 4,\r
+ 6,\r
(VOID**)&Private->Buffer,\r
0\r
);\r
goto Exit;\r
}\r
\r
- Bytes = EFI_PAGES_TO_SIZE (4);\r
+ Bytes = EFI_PAGES_TO_SIZE (6);\r
Status = PciIo->Map (\r
PciIo,\r
EfiPciIoOperationBusMasterCommonBuffer,\r
&Private->Mapping\r
);\r
\r
- if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (4))) {\r
+ if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (6))) {\r
goto Exit;\r
}\r
\r
Private->BufferPciAddr = (UINT8 *)(UINTN)MappedAddr;\r
- ZeroMem (Private->Buffer, EFI_PAGES_TO_SIZE (4));\r
\r
Private->Signature = NVME_CONTROLLER_PRIVATE_DATA_SIGNATURE;\r
Private->ControllerHandle = Controller;\r
Private->Passthru.BuildDevicePath = NvmExpressBuildDevicePath;\r
Private->Passthru.GetNamespace = NvmExpressGetNamespace;\r
CopyMem (&Private->PassThruMode, &gEfiNvmExpressPassThruMode, sizeof (EFI_NVM_EXPRESS_PASS_THRU_MODE));\r
+ InitializeListHead (&Private->AsyncPassThruQueue);\r
+ InitializeListHead (&Private->UnsubmittedSubtasks);\r
\r
Status = NvmeControllerInit (Private);\r
if (EFI_ERROR(Status)) {\r
goto Exit;\r
}\r
\r
+ //\r
+ // Start the asynchronous I/O completion monitor\r
+ //\r
+ Status = gBS->CreateEvent (\r
+ EVT_TIMER | EVT_NOTIFY_SIGNAL,\r
+ TPL_NOTIFY,\r
+ ProcessAsyncTaskList,\r
+ Private,\r
+ &Private->TimerEvent\r
+ );\r
+ if (EFI_ERROR (Status)) {\r
+ goto Exit;\r
+ }\r
+\r
+ Status = gBS->SetTimer (\r
+ Private->TimerEvent,\r
+ TimerPeriodic,\r
+ NVME_HC_ASYNC_TIMER\r
+ );\r
+ if (EFI_ERROR (Status)) {\r
+ goto Exit;\r
+ }\r
+\r
Status = gBS->InstallMultipleProtocolInterfaces (\r
&Controller,\r
&gEfiNvmExpressPassThruProtocolGuid,\r
if (EFI_ERROR (Status)) {\r
goto Exit;\r
}\r
+\r
+ NvmeRegisterShutdownNotification ();\r
} else {\r
Status = gBS->OpenProtocol (\r
Controller,\r
}\r
\r
if ((Private != NULL) && (Private->Buffer != NULL)) {\r
- PciIo->FreeBuffer (PciIo, 4, Private->Buffer);\r
+ PciIo->FreeBuffer (PciIo, 6, Private->Buffer);\r
+ }\r
+\r
+ if ((Private != NULL) && (Private->ControllerData != NULL)) {\r
+ FreePool (Private->ControllerData);\r
}\r
\r
if (Private != NULL) {\r
+ if (Private->TimerEvent != NULL) {\r
+ gBS->CloseEvent (Private->TimerEvent);\r
+ }\r
+\r
FreePool (Private);\r
}\r
\r
UINTN Index;\r
NVME_CONTROLLER_PRIVATE_DATA *Private;\r
EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *PassThru;\r
+ BOOLEAN IsEmpty;\r
+ EFI_TPL OldTpl;\r
\r
if (NumberOfChildren == 0) {\r
Status = gBS->OpenProtocol (\r
\r
if (!EFI_ERROR (Status)) {\r
Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (PassThru);\r
+\r
+ //\r
+ // Wait for the asynchronous PassThru queue to become empty.\r
+ //\r
+ while (TRUE) {\r
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);\r
+ IsEmpty = IsListEmpty (&Private->AsyncPassThruQueue) &&\r
+ IsListEmpty (&Private->UnsubmittedSubtasks);\r
+ gBS->RestoreTPL (OldTpl);\r
+\r
+ if (IsEmpty) {\r
+ break;\r
+ }\r
+\r
+ gBS->Stall (100);\r
+ }\r
+\r
gBS->UninstallMultipleProtocolInterfaces (\r
Controller,\r
&gEfiNvmExpressPassThruProtocolGuid,\r
NULL\r
);\r
\r
+ if (Private->TimerEvent != NULL) {\r
+ gBS->CloseEvent (Private->TimerEvent);\r
+ }\r
+\r
if (Private->Mapping != NULL) {\r
Private->PciIo->Unmap (Private->PciIo, Private->Mapping);\r
}\r
\r
if (Private->Buffer != NULL) {\r
- Private->PciIo->FreeBuffer (Private->PciIo, 4, Private->Buffer);\r
+ Private->PciIo->FreeBuffer (Private->PciIo, 6, Private->Buffer);\r
}\r
\r
FreePool (Private->ControllerData);\r
This->DriverBindingHandle,\r
Controller\r
);\r
+\r
+ NvmeUnregisterShutdownNotification ();\r
+\r
return EFI_SUCCESS;\r
}\r
\r
projects / mirror_edk2.git / blobdiff