Device->BlockIo.WriteBlocks = NvmeBlockIoWriteBlocks;\r
Device->BlockIo.FlushBlocks = NvmeBlockIoFlushBlocks;\r
\r
+ //\r
+ // Create BlockIo2 Protocol instance\r
+ //\r
+ Device->BlockIo2.Media = &Device->Media;\r
+ Device->BlockIo2.Reset = NvmeBlockIoResetEx;\r
+ Device->BlockIo2.ReadBlocksEx = NvmeBlockIoReadBlocksEx;\r
+ Device->BlockIo2.WriteBlocksEx = NvmeBlockIoWriteBlocksEx;\r
+ Device->BlockIo2.FlushBlocksEx = NvmeBlockIoFlushBlocksEx;\r
+ InitializeListHead (&Device->AsyncQueue);\r
+\r
//\r
// Create StorageSecurityProtocol Instance\r
//\r
Device->DevicePath,\r
&gEfiBlockIoProtocolGuid,\r
&Device->BlockIo,\r
+ &gEfiBlockIo2ProtocolGuid,\r
+ &Device->BlockIo2,\r
&gEfiDiskInfoProtocolGuid,\r
&Device->DiskInfo,\r
NULL\r
Device->DevicePath,\r
&gEfiBlockIoProtocolGuid,\r
&Device->BlockIo,\r
+ &gEfiBlockIo2ProtocolGuid,\r
+ &Device->BlockIo2,\r
&gEfiDiskInfoProtocolGuid,\r
&Device->DiskInfo,\r
NULL\r
NVME_DEVICE_PRIVATE_DATA *Device;\r
NVME_CONTROLLER_PRIVATE_DATA *Private;\r
EFI_STORAGE_SECURITY_COMMAND_PROTOCOL *StorageSecurity;\r
+ BOOLEAN IsEmpty;\r
+ EFI_TPL OldTpl;\r
\r
BlockIo = NULL;\r
\r
Device = NVME_DEVICE_PRIVATE_DATA_FROM_BLOCK_IO (BlockIo);\r
Private = Device->Controller;\r
\r
+ //\r
+ // Wait for the device's asynchronous I/O queue to become empty.\r
+ //\r
+ while (TRUE) {\r
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);\r
+ IsEmpty = IsListEmpty (&Device->AsyncQueue);\r
+ gBS->RestoreTPL (OldTpl);\r
+\r
+ if (IsEmpty) {\r
+ break;\r
+ }\r
+\r
+ gBS->Stall (100);\r
+ }\r
+\r
//\r
// Close the child handle\r
//\r
Device->DevicePath,\r
&gEfiBlockIoProtocolGuid,\r
&Device->BlockIo,\r
+ &gEfiBlockIo2ProtocolGuid,\r
+ &Device->BlockIo2,\r
&gEfiDiskInfoProtocolGuid,\r
&Device->DiskInfo,\r
NULL\r
return EFI_SUCCESS;\r
}\r
\r
+/**\r
+ Call back function when the timer event is signaled.\r
+\r
+ @param[in] Event The Event this notify function registered to.\r
+ @param[in] Context Pointer to the context data registered to the\r
+ Event.\r
+\r
+**/\r
+VOID\r
+EFIAPI\r
+ProcessAsyncTaskList (\r
+ IN EFI_EVENT Event,\r
+ IN VOID* Context\r
+ )\r
+{\r
+ NVME_CONTROLLER_PRIVATE_DATA *Private;\r
+ EFI_PCI_IO_PROTOCOL *PciIo;\r
+ NVME_CQ *Cq;\r
+ UINT16 QueueId;\r
+ UINT32 Data;\r
+ LIST_ENTRY *Link;\r
+ LIST_ENTRY *NextLink;\r
+ NVME_PASS_THRU_ASYNC_REQ *AsyncRequest;\r
+ NVME_BLKIO2_SUBTASK *Subtask;\r
+ NVME_BLKIO2_REQUEST *BlkIo2Request;\r
+ EFI_BLOCK_IO2_TOKEN *Token;\r
+ BOOLEAN HasNewItem;\r
+ EFI_STATUS Status;\r
+\r
+ Private = (NVME_CONTROLLER_PRIVATE_DATA*)Context;\r
+ QueueId = 2;\r
+ Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;\r
+ HasNewItem = FALSE;\r
+\r
+ //\r
+ // Submit asynchronous subtasks to the NVMe Submission Queue\r
+ //\r
+ for (Link = GetFirstNode (&Private->UnsubmittedSubtasks);\r
+ !IsNull (&Private->UnsubmittedSubtasks, Link);\r
+ Link = NextLink) {\r
+ NextLink = GetNextNode (&Private->UnsubmittedSubtasks, Link);\r
+ Subtask = NVME_BLKIO2_SUBTASK_FROM_LINK (Link);\r
+ BlkIo2Request = Subtask->BlockIo2Request;\r
+ Token = BlkIo2Request->Token;\r
+ RemoveEntryList (Link);\r
+ BlkIo2Request->UnsubmittedSubtaskNum--;\r
+\r
+ //\r
+ // If any previous subtask fails, do not process subsequent ones.\r
+ //\r
+ if (Token->TransactionStatus != EFI_SUCCESS) {\r
+ if (IsListEmpty (&BlkIo2Request->SubtasksQueue) &&\r
+ BlkIo2Request->LastSubtaskSubmitted &&\r
+ (BlkIo2Request->UnsubmittedSubtaskNum == 0)) {\r
+ //\r
+ // Remove the BlockIo2 request from the device asynchronous queue.\r
+ //\r
+ RemoveEntryList (&BlkIo2Request->Link);\r
+ FreePool (BlkIo2Request);\r
+ gBS->SignalEvent (Token->Event);\r
+ }\r
+\r
+ FreePool (Subtask->CommandPacket->NvmeCmd);\r
+ FreePool (Subtask->CommandPacket->NvmeCompletion);\r
+ FreePool (Subtask->CommandPacket);\r
+ FreePool (Subtask);\r
+\r
+ continue;\r
+ }\r
+\r
+ Status = Private->Passthru.PassThru (\r
+ &Private->Passthru,\r
+ Subtask->NamespaceId,\r
+ Subtask->CommandPacket,\r
+ Subtask->Event\r
+ );\r
+ if (Status == EFI_NOT_READY) {\r
+ InsertHeadList (&Private->UnsubmittedSubtasks, Link);\r
+ BlkIo2Request->UnsubmittedSubtaskNum++;\r
+ break;\r
+ } else if (EFI_ERROR (Status)) {\r
+ Token->TransactionStatus = EFI_DEVICE_ERROR;\r
+\r
+ if (IsListEmpty (&BlkIo2Request->SubtasksQueue) &&\r
+ Subtask->IsLast) {\r
+ //\r
+ // Remove the BlockIo2 request from the device asynchronous queue.\r
+ //\r
+ RemoveEntryList (&BlkIo2Request->Link);\r
+ FreePool (BlkIo2Request);\r
+ gBS->SignalEvent (Token->Event);\r
+ }\r
+\r
+ FreePool (Subtask->CommandPacket->NvmeCmd);\r
+ FreePool (Subtask->CommandPacket->NvmeCompletion);\r
+ FreePool (Subtask->CommandPacket);\r
+ FreePool (Subtask);\r
+ } else {\r
+ InsertTailList (&BlkIo2Request->SubtasksQueue, Link);\r
+ if (Subtask->IsLast) {\r
+ BlkIo2Request->LastSubtaskSubmitted = TRUE;\r
+ }\r
+ }\r
+ }\r
+\r
+ while (Cq->Pt != Private->Pt[QueueId]) {\r
+ ASSERT (Cq->Sqid == QueueId);\r
+\r
+ HasNewItem = TRUE;\r
+\r
+ //\r
+ // Find the command with given Command Id.\r
+ //\r
+ for (Link = GetFirstNode (&Private->AsyncPassThruQueue);\r
+ !IsNull (&Private->AsyncPassThruQueue, Link);\r
+ Link = NextLink) {\r
+ NextLink = GetNextNode (&Private->AsyncPassThruQueue, Link);\r
+ AsyncRequest = NVME_PASS_THRU_ASYNC_REQ_FROM_THIS (Link);\r
+ if (AsyncRequest->CommandId == Cq->Cid) {\r
+ //\r
+ // Copy the Respose Queue entry for this command to the callers\r
+ // response buffer.\r
+ //\r
+ CopyMem (\r
+ AsyncRequest->Packet->NvmeCompletion,\r
+ Cq,\r
+ sizeof(EFI_NVM_EXPRESS_COMPLETION)\r
+ );\r
+\r
+ RemoveEntryList (Link);\r
+ gBS->SignalEvent (AsyncRequest->CallerEvent);\r
+ FreePool (AsyncRequest);\r
+\r
+ //\r
+ // Update submission queue head.\r
+ //\r
+ Private->AsyncSqHead = Cq->Sqhd;\r
+ break;\r
+ }\r
+ }\r
+\r
+ Private->CqHdbl[QueueId].Cqh++;\r
+ if (Private->CqHdbl[QueueId].Cqh > NVME_ASYNC_CCQ_SIZE) {\r
+ Private->CqHdbl[QueueId].Cqh = 0;\r
+ Private->Pt[QueueId] ^= 1;\r
+ }\r
+\r
+ Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;\r
+ }\r
+\r
+ if (HasNewItem) {\r
+ PciIo = Private->PciIo;\r
+ Data = ReadUnaligned32 ((UINT32*)&Private->CqHdbl[QueueId]);\r
+ PciIo->Mem.Write (\r
+ PciIo,\r
+ EfiPciIoWidthUint32,\r
+ NVME_BAR,\r
+ NVME_CQHDBL_OFFSET(QueueId, Private->Cap.Dstrd),\r
+ 1,\r
+ &Data\r
+ );\r
+ }\r
+}\r
+\r
/**\r
Tests to see if this driver supports a given controller. If a child device is provided,\r
it further tests to see if this driver supports creating a handle for the specified child device.\r
}\r
\r
//\r
- // 4 x 4kB aligned buffers will be carved out of this buffer.\r
+ // 6 x 4kB aligned buffers will be carved out of this buffer.\r
// 1st 4kB boundary is the start of the admin submission queue.\r
// 2nd 4kB boundary is the start of the admin completion queue.\r
// 3rd 4kB boundary is the start of I/O submission queue #1.\r
// 4th 4kB boundary is the start of I/O completion queue #1.\r
+ // 5th 4kB boundary is the start of I/O submission queue #2.\r
+ // 6th 4kB boundary is the start of I/O completion queue #2.\r
//\r
- // Allocate 4 pages of memory, then map it for bus master read and write.\r
+ // Allocate 6 pages of memory, then map it for bus master read and write.\r
//\r
Status = PciIo->AllocateBuffer (\r
PciIo,\r
AllocateAnyPages,\r
EfiBootServicesData,\r
- 4,\r
+ 6,\r
(VOID**)&Private->Buffer,\r
0\r
);\r
goto Exit;\r
}\r
\r
- Bytes = EFI_PAGES_TO_SIZE (4);\r
+ Bytes = EFI_PAGES_TO_SIZE (6);\r
Status = PciIo->Map (\r
PciIo,\r
EfiPciIoOperationBusMasterCommonBuffer,\r
&Private->Mapping\r
);\r
\r
- if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (4))) {\r
+ if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (6))) {\r
goto Exit;\r
}\r
\r
Private->Passthru.BuildDevicePath = NvmExpressBuildDevicePath;\r
Private->Passthru.GetNamespace = NvmExpressGetNamespace;\r
CopyMem (&Private->PassThruMode, &gEfiNvmExpressPassThruMode, sizeof (EFI_NVM_EXPRESS_PASS_THRU_MODE));\r
+ InitializeListHead (&Private->AsyncPassThruQueue);\r
+ InitializeListHead (&Private->UnsubmittedSubtasks);\r
\r
Status = NvmeControllerInit (Private);\r
if (EFI_ERROR(Status)) {\r
goto Exit;\r
}\r
\r
+ //\r
+ // Start the asynchronous I/O completion monitor\r
+ //\r
+ Status = gBS->CreateEvent (\r
+ EVT_TIMER | EVT_NOTIFY_SIGNAL,\r
+ TPL_NOTIFY,\r
+ ProcessAsyncTaskList,\r
+ Private,\r
+ &Private->TimerEvent\r
+ );\r
+ if (EFI_ERROR (Status)) {\r
+ goto Exit;\r
+ }\r
+\r
+ Status = gBS->SetTimer (\r
+ Private->TimerEvent,\r
+ TimerPeriodic,\r
+ NVME_HC_ASYNC_TIMER\r
+ );\r
+ if (EFI_ERROR (Status)) {\r
+ goto Exit;\r
+ }\r
+\r
Status = gBS->InstallMultipleProtocolInterfaces (\r
&Controller,\r
&gEfiNvmExpressPassThruProtocolGuid,\r
}\r
\r
if ((Private != NULL) && (Private->Buffer != NULL)) {\r
- PciIo->FreeBuffer (PciIo, 4, Private->Buffer);\r
+ PciIo->FreeBuffer (PciIo, 6, Private->Buffer);\r
}\r
\r
if ((Private != NULL) && (Private->ControllerData != NULL)) {\r
}\r
\r
if (Private != NULL) {\r
+ if (Private->TimerEvent != NULL) {\r
+ gBS->CloseEvent (Private->TimerEvent);\r
+ }\r
+\r
FreePool (Private);\r
}\r
\r
UINTN Index;\r
NVME_CONTROLLER_PRIVATE_DATA *Private;\r
EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *PassThru;\r
+ BOOLEAN IsEmpty;\r
+ EFI_TPL OldTpl;\r
\r
if (NumberOfChildren == 0) {\r
Status = gBS->OpenProtocol (\r
\r
if (!EFI_ERROR (Status)) {\r
Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (PassThru);\r
+\r
+ //\r
+ // Wait for the asynchronous PassThru queue to become empty.\r
+ //\r
+ while (TRUE) {\r
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);\r
+ IsEmpty = IsListEmpty (&Private->AsyncPassThruQueue) &&\r
+ IsListEmpty (&Private->UnsubmittedSubtasks);\r
+ gBS->RestoreTPL (OldTpl);\r
+\r
+ if (IsEmpty) {\r
+ break;\r
+ }\r
+\r
+ gBS->Stall (100);\r
+ }\r
+\r
gBS->UninstallMultipleProtocolInterfaces (\r
Controller,\r
&gEfiNvmExpressPassThruProtocolGuid,\r
NULL\r
);\r
\r
+ if (Private->TimerEvent != NULL) {\r
+ gBS->CloseEvent (Private->TimerEvent);\r
+ }\r
+\r
if (Private->Mapping != NULL) {\r
Private->PciIo->Unmap (Private->PciIo, Private->Mapping);\r
}\r
\r
if (Private->Buffer != NULL) {\r
- Private->PciIo->FreeBuffer (Private->PciIo, 4, Private->Buffer);\r
+ Private->PciIo->FreeBuffer (Private->PciIo, 6, Private->Buffer);\r
}\r
\r
FreePool (Private->ControllerData);\r
projects / mirror_edk2.git / blobdiff