+/**\r
+ Nonblocking I/O callback funtion when the event is signaled.\r
+\r
+ @param[in] Event The Event this notify function registered to.\r
+ @param[in] Context Pointer to the context data registered to the\r
+ Event.\r
+\r
+**/\r
+VOID\r
+EFIAPI\r
+AsyncIoCallback (\r
+ IN EFI_EVENT Event,\r
+ IN VOID *Context\r
+ )\r
+{\r
+ NVME_BLKIO2_SUBTASK *Subtask;\r
+ NVME_BLKIO2_REQUEST *Request;\r
+ NVME_CQ *Completion;\r
+ EFI_BLOCK_IO2_TOKEN *Token;\r
+\r
+ gBS->CloseEvent (Event);\r
+\r
+ Subtask = (NVME_BLKIO2_SUBTASK *) Context;\r
+ Completion = (NVME_CQ *) Subtask->CommandPacket->NvmeCompletion;\r
+ Request = Subtask->BlockIo2Request;\r
+ Token = Request->Token;\r
+\r
+ if (Token->TransactionStatus == EFI_SUCCESS) {\r
+ //\r
+ // If previous subtask already fails, do not check the result of\r
+ // subsequent subtasks.\r
+ //\r
+ if ((Completion->Sct != 0) || (Completion->Sc != 0)) {\r
+ Token->TransactionStatus = EFI_DEVICE_ERROR;\r
+\r
+ //\r
+ // Dump completion entry status for debugging.\r
+ //\r
+ DEBUG_CODE_BEGIN();\r
+ NvmeDumpStatus (Completion);\r
+ DEBUG_CODE_END();\r
+ }\r
+ }\r
+\r
+ //\r
+ // Remove the subtask from the BlockIo2 subtasks list.\r
+ //\r
+ RemoveEntryList (&Subtask->Link);\r
+\r
+ if (IsListEmpty (&Request->SubtasksQueue) && Request->LastSubtaskSubmitted) {\r
+ //\r
+ // Remove the BlockIo2 request from the device asynchronous queue.\r
+ //\r
+ RemoveEntryList (&Request->Link);\r
+ FreePool (Request);\r
+ gBS->SignalEvent (Token->Event);\r
+ }\r
+\r
+ FreePool (Subtask->CommandPacket->NvmeCmd);\r
+ FreePool (Subtask->CommandPacket->NvmeCompletion);\r
+ FreePool (Subtask->CommandPacket);\r
+ FreePool (Subtask);\r
+}\r
+\r
+/**\r
+ Read some sectors from the device in an asynchronous manner.\r
+\r
+ @param Device The pointer to the NVME_DEVICE_PRIVATE_DATA data\r
+ structure.\r
+ @param Request The pointer to the NVME_BLKIO2_REQUEST data structure.\r
+ @param Buffer The buffer used to store the data read from the device.\r
+ @param Lba The start block number.\r
+ @param Blocks Total block number to be read.\r
+ @param IsLast The last subtask of an asynchronous read request.\r
+\r
+ @retval EFI_SUCCESS Asynchronous read request has been queued.\r
+ @retval Others Fail to send the asynchronous request.\r
+\r
+**/\r
+EFI_STATUS\r
+AsyncReadSectors (\r
+ IN NVME_DEVICE_PRIVATE_DATA *Device,\r
+ IN NVME_BLKIO2_REQUEST *Request,\r
+ IN UINT64 Buffer,\r
+ IN UINT64 Lba,\r
+ IN UINT32 Blocks,\r
+ IN BOOLEAN IsLast\r
+ )\r
+{\r
+ NVME_CONTROLLER_PRIVATE_DATA *Private;\r
+ UINT32 Bytes;\r
+ NVME_BLKIO2_SUBTASK *Subtask;\r
+ EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET *CommandPacket;\r
+ EFI_NVM_EXPRESS_COMMAND *Command;\r
+ EFI_NVM_EXPRESS_COMPLETION *Completion;\r
+ EFI_STATUS Status;\r
+ UINT32 BlockSize;\r
+ EFI_TPL OldTpl;\r
+\r
+ Private = Device->Controller;\r
+ BlockSize = Device->Media.BlockSize;\r
+ Bytes = Blocks * BlockSize;\r
+ CommandPacket = NULL;\r
+ Command = NULL;\r
+ Completion = NULL;\r
+\r
+ Subtask = AllocateZeroPool (sizeof (NVME_BLKIO2_SUBTASK));\r
+ if (Subtask == NULL) {\r
+ Status = EFI_OUT_OF_RESOURCES;\r
+ goto ErrorExit;\r
+ }\r
+\r
+ Subtask->Signature = NVME_BLKIO2_SUBTASK_SIGNATURE;\r
+ Subtask->IsLast = IsLast;\r
+ Subtask->NamespaceId = Device->NamespaceId;\r
+ Subtask->BlockIo2Request = Request;\r
+\r
+ CommandPacket = AllocateZeroPool (sizeof (EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));\r
+ if (CommandPacket == NULL) {\r
+ Status = EFI_OUT_OF_RESOURCES;\r
+ goto ErrorExit;\r
+ } else {\r
+ Subtask->CommandPacket = CommandPacket;\r
+ }\r
+\r
+ Command = AllocateZeroPool (sizeof (EFI_NVM_EXPRESS_COMMAND));\r
+ if (Command == NULL) {\r
+ Status = EFI_OUT_OF_RESOURCES;\r
+ goto ErrorExit;\r
+ }\r
+\r
+ Completion = AllocateZeroPool (sizeof (EFI_NVM_EXPRESS_COMPLETION));\r
+ if (Completion == NULL) {\r
+ Status = EFI_OUT_OF_RESOURCES;\r
+ goto ErrorExit;\r
+ }\r
+\r
+ //\r
+ // Create Event\r
+ //\r
+ Status = gBS->CreateEvent (\r
+ EVT_NOTIFY_SIGNAL,\r
+ TPL_NOTIFY,\r
+ AsyncIoCallback,\r
+ Subtask,\r
+ &Subtask->Event\r
+ );\r
+ if (EFI_ERROR(Status)) {\r
+ goto ErrorExit;\r
+ }\r
+\r
+ CommandPacket->NvmeCmd = Command;\r
+ CommandPacket->NvmeCompletion = Completion;\r
+\r
+ CommandPacket->NvmeCmd->Cdw0.Opcode = NVME_IO_READ_OPC;\r
+ CommandPacket->NvmeCmd->Nsid = Device->NamespaceId;\r
+ CommandPacket->TransferBuffer = (VOID *)(UINTN)Buffer;\r
+\r
+ CommandPacket->TransferLength = Bytes;\r
+ CommandPacket->CommandTimeout = NVME_GENERIC_TIMEOUT;\r
+ CommandPacket->QueueType = NVME_IO_QUEUE;\r
+\r
+ CommandPacket->NvmeCmd->Cdw10 = (UINT32)Lba;\r
+ CommandPacket->NvmeCmd->Cdw11 = (UINT32)RShiftU64(Lba, 32);\r
+ CommandPacket->NvmeCmd->Cdw12 = (Blocks - 1) & 0xFFFF;\r
+\r
+ CommandPacket->NvmeCmd->Flags = CDW10_VALID | CDW11_VALID | CDW12_VALID;\r
+\r
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);\r
+ InsertTailList (&Private->UnsubmittedSubtasks, &Subtask->Link);\r
+ Request->UnsubmittedSubtaskNum++;\r
+ gBS->RestoreTPL (OldTpl);\r
+\r
+ return EFI_SUCCESS;\r
+\r
+ErrorExit:\r
+ //\r
+ // Resource cleanup if asynchronous read request has not been queued.\r
+ //\r
+ if (Completion != NULL) {\r
+ FreePool (Completion);\r
+ }\r
+\r
+ if (Command != NULL) {\r
+ FreePool (Command);\r
+ }\r
+\r
+ if (CommandPacket != NULL) {\r
+ FreePool (CommandPacket);\r
+ }\r
+\r
+ if (Subtask != NULL) {\r
+ if (Subtask->Event != NULL) {\r
+ gBS->CloseEvent (Subtask->Event);\r
+ }\r
+\r
+ FreePool (Subtask);\r
+ }\r
+\r
+ return Status;\r
+}\r
+\r
+/**\r
+ Write some sectors from the device in an asynchronous manner.\r
+\r
+ @param Device The pointer to the NVME_DEVICE_PRIVATE_DATA data\r
+ structure.\r
+ @param Request The pointer to the NVME_BLKIO2_REQUEST data structure.\r
+ @param Buffer The buffer used to store the data written to the\r
+ device.\r
+ @param Lba The start block number.\r
+ @param Blocks Total block number to be written.\r
+ @param IsLast The last subtask of an asynchronous write request.\r
+\r
+ @retval EFI_SUCCESS Asynchronous write request has been queued.\r
+ @retval Others Fail to send the asynchronous request.\r
+\r
+**/\r
+EFI_STATUS\r
+AsyncWriteSectors (\r
+ IN NVME_DEVICE_PRIVATE_DATA *Device,\r
+ IN NVME_BLKIO2_REQUEST *Request,\r
+ IN UINT64 Buffer,\r
+ IN UINT64 Lba,\r
+ IN UINT32 Blocks,\r
+ IN BOOLEAN IsLast\r
+ )\r
+{\r
+ NVME_CONTROLLER_PRIVATE_DATA *Private;\r
+ UINT32 Bytes;\r
+ NVME_BLKIO2_SUBTASK *Subtask;\r
+ EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET *CommandPacket;\r
+ EFI_NVM_EXPRESS_COMMAND *Command;\r
+ EFI_NVM_EXPRESS_COMPLETION *Completion;\r
+ EFI_STATUS Status;\r
+ UINT32 BlockSize;\r
+ EFI_TPL OldTpl;\r
+\r
+ Private = Device->Controller;\r
+ BlockSize = Device->Media.BlockSize;\r
+ Bytes = Blocks * BlockSize;\r
+ CommandPacket = NULL;\r
+ Command = NULL;\r
+ Completion = NULL;\r
+\r
+ Subtask = AllocateZeroPool (sizeof (NVME_BLKIO2_SUBTASK));\r
+ if (Subtask == NULL) {\r
+ Status = EFI_OUT_OF_RESOURCES;\r
+ goto ErrorExit;\r
+ }\r
+\r
+ Subtask->Signature = NVME_BLKIO2_SUBTASK_SIGNATURE;\r
+ Subtask->IsLast = IsLast;\r
+ Subtask->NamespaceId = Device->NamespaceId;\r
+ Subtask->BlockIo2Request = Request;\r
+\r
+ CommandPacket = AllocateZeroPool (sizeof (EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));\r
+ if (CommandPacket == NULL) {\r
+ Status = EFI_OUT_OF_RESOURCES;\r
+ goto ErrorExit;\r
+ } else {\r
+ Subtask->CommandPacket = CommandPacket;\r
+ }\r
+\r
+ Command = AllocateZeroPool (sizeof (EFI_NVM_EXPRESS_COMMAND));\r
+ if (Command == NULL) {\r
+ Status = EFI_OUT_OF_RESOURCES;\r
+ goto ErrorExit;\r
+ }\r
+\r
+ Completion = AllocateZeroPool (sizeof (EFI_NVM_EXPRESS_COMPLETION));\r
+ if (Completion == NULL) {\r
+ Status = EFI_OUT_OF_RESOURCES;\r
+ goto ErrorExit;\r
+ }\r
+\r
+ //\r
+ // Create Event\r
+ //\r
+ Status = gBS->CreateEvent (\r
+ EVT_NOTIFY_SIGNAL,\r
+ TPL_NOTIFY,\r
+ AsyncIoCallback,\r
+ Subtask,\r
+ &Subtask->Event\r
+ );\r
+ if (EFI_ERROR(Status)) {\r
+ goto ErrorExit;\r
+ }\r
+\r
+ CommandPacket->NvmeCmd = Command;\r
+ CommandPacket->NvmeCompletion = Completion;\r
+\r
+ CommandPacket->NvmeCmd->Cdw0.Opcode = NVME_IO_WRITE_OPC;\r
+ CommandPacket->NvmeCmd->Nsid = Device->NamespaceId;\r
+ CommandPacket->TransferBuffer = (VOID *)(UINTN)Buffer;\r
+\r
+ CommandPacket->TransferLength = Bytes;\r
+ CommandPacket->CommandTimeout = NVME_GENERIC_TIMEOUT;\r
+ CommandPacket->QueueType = NVME_IO_QUEUE;\r
+\r
+ CommandPacket->NvmeCmd->Cdw10 = (UINT32)Lba;\r
+ CommandPacket->NvmeCmd->Cdw11 = (UINT32)RShiftU64(Lba, 32);\r
+ //\r
+ // Set Force Unit Access bit (bit 30) to use write-through behaviour\r
+ //\r
+ CommandPacket->NvmeCmd->Cdw12 = ((Blocks - 1) & 0xFFFF) | BIT30;\r
+\r
+ CommandPacket->NvmeCmd->Flags = CDW10_VALID | CDW11_VALID | CDW12_VALID;\r
+\r
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);\r
+ InsertTailList (&Private->UnsubmittedSubtasks, &Subtask->Link);\r
+ Request->UnsubmittedSubtaskNum++;\r
+ gBS->RestoreTPL (OldTpl);\r
+\r
+ return EFI_SUCCESS;\r
+\r
+ErrorExit:\r
+ //\r
+ // Resource cleanup if asynchronous read request has not been queued.\r
+ //\r
+ if (Completion != NULL) {\r
+ FreePool (Completion);\r
+ }\r
+\r
+ if (Command != NULL) {\r
+ FreePool (Command);\r
+ }\r
+\r
+ if (CommandPacket != NULL) {\r
+ FreePool (CommandPacket);\r
+ }\r
+\r
+ if (Subtask != NULL) {\r
+ if (Subtask->Event != NULL) {\r
+ gBS->CloseEvent (Subtask->Event);\r
+ }\r
+\r
+ FreePool (Subtask);\r
+ }\r
+\r
+ return Status;\r
+}\r
+\r
+/**\r
+ Read some blocks from the device in an asynchronous manner.\r
+\r
+ @param Device The pointer to the NVME_DEVICE_PRIVATE_DATA data\r
+ structure.\r
+ @param Buffer The buffer used to store the data read from the device.\r
+ @param Lba The start block number.\r
+ @param Blocks Total block number to be read.\r
+ @param Token A pointer to the token associated with the transaction.\r
+\r
+ @retval EFI_SUCCESS Data are read from the device.\r
+ @retval Others Fail to read all the data.\r
+\r
+**/\r
+EFI_STATUS\r
+NvmeAsyncRead (\r
+ IN NVME_DEVICE_PRIVATE_DATA *Device,\r
+ OUT VOID *Buffer,\r
+ IN UINT64 Lba,\r
+ IN UINTN Blocks,\r
+ IN EFI_BLOCK_IO2_TOKEN *Token\r
+ )\r
+{\r
+ EFI_STATUS Status;\r
+ UINT32 BlockSize;\r
+ NVME_CONTROLLER_PRIVATE_DATA *Private;\r
+ NVME_BLKIO2_REQUEST *BlkIo2Req;\r
+ UINT32 MaxTransferBlocks;\r
+ UINTN OrginalBlocks;\r
+ BOOLEAN IsEmpty;\r
+ EFI_TPL OldTpl;\r
+\r
+ Status = EFI_SUCCESS;\r
+ Private = Device->Controller;\r
+ BlockSize = Device->Media.BlockSize;\r
+ OrginalBlocks = Blocks;\r
+ BlkIo2Req = AllocateZeroPool (sizeof (NVME_BLKIO2_REQUEST));\r
+ if (BlkIo2Req == NULL) {\r
+ return EFI_OUT_OF_RESOURCES;\r
+ }\r
+\r
+ BlkIo2Req->Signature = NVME_BLKIO2_REQUEST_SIGNATURE;\r
+ BlkIo2Req->Token = Token;\r
+\r
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);\r
+ InsertTailList (&Device->AsyncQueue, &BlkIo2Req->Link);\r
+ gBS->RestoreTPL (OldTpl);\r
+\r
+ InitializeListHead (&BlkIo2Req->SubtasksQueue);\r
+\r
+ if (Private->ControllerData->Mdts != 0) {\r
+ MaxTransferBlocks = (1 << (Private->ControllerData->Mdts)) * (1 << (Private->Cap.Mpsmin + 12)) / BlockSize;\r
+ } else {\r
+ MaxTransferBlocks = 1024;\r
+ }\r
+\r
+ while (Blocks > 0) {\r
+ if (Blocks > MaxTransferBlocks) {\r
+ Status = AsyncReadSectors (\r
+ Device,\r
+ BlkIo2Req, (UINT64)(UINTN)Buffer,\r
+ Lba,\r
+ MaxTransferBlocks,\r
+ FALSE\r
+ );\r
+\r
+ Blocks -= MaxTransferBlocks;\r
+ Buffer = (VOID *)(UINTN)((UINT64)(UINTN)Buffer + MaxTransferBlocks * BlockSize);\r
+ Lba += MaxTransferBlocks;\r
+ } else {\r
+ Status = AsyncReadSectors (\r
+ Device,\r
+ BlkIo2Req,\r
+ (UINT64)(UINTN)Buffer,\r
+ Lba,\r
+ (UINT32)Blocks,\r
+ TRUE\r
+ );\r
+\r
+ Blocks = 0;\r
+ }\r
+\r
+ if (EFI_ERROR(Status)) {\r
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);\r
+ IsEmpty = IsListEmpty (&BlkIo2Req->SubtasksQueue) &&\r
+ (BlkIo2Req->UnsubmittedSubtaskNum == 0);\r
+\r
+ if (IsEmpty) {\r
+ //\r
+ // Remove the BlockIo2 request from the device asynchronous queue.\r
+ //\r
+ RemoveEntryList (&BlkIo2Req->Link);\r
+ FreePool (BlkIo2Req);\r
+ Status = EFI_DEVICE_ERROR;\r
+ } else {\r
+ //\r
+ // There are previous BlockIo2 subtasks still running, EFI_SUCCESS\r
+ // should be returned to make sure that the caller does not free\r
+ // resources still using by these requests.\r
+ //\r
+ Status = EFI_SUCCESS;\r
+ Token->TransactionStatus = EFI_DEVICE_ERROR;\r
+ BlkIo2Req->LastSubtaskSubmitted = TRUE;\r
+ }\r
+\r
+ gBS->RestoreTPL (OldTpl);\r
+\r
+ break;\r
+ }\r
+ }\r
+\r
+ DEBUG ((EFI_D_VERBOSE, "%a: Lba = 0x%08Lx, Original = 0x%08Lx, "\r
+ "Remaining = 0x%08Lx, BlockSize = 0x%x, Status = %r\n", __FUNCTION__, Lba,\r
+ (UINT64)OrginalBlocks, (UINT64)Blocks, BlockSize, Status));\r
+\r
+ return Status;\r
+}\r
+\r
+/**\r
+ Write some blocks from the device in an asynchronous manner.\r
+\r
+ @param Device The pointer to the NVME_DEVICE_PRIVATE_DATA data\r
+ structure.\r
+ @param Buffer The buffer used to store the data written to the\r
+ device.\r
+ @param Lba The start block number.\r
+ @param Blocks Total block number to be written.\r
+ @param Token A pointer to the token associated with the transaction.\r
+\r
+ @retval EFI_SUCCESS Data are written to the device.\r
+ @retval Others Fail to write all the data.\r
+\r
+**/\r
+EFI_STATUS\r
+NvmeAsyncWrite (\r
+ IN NVME_DEVICE_PRIVATE_DATA *Device,\r
+ IN VOID *Buffer,\r
+ IN UINT64 Lba,\r
+ IN UINTN Blocks,\r
+ IN EFI_BLOCK_IO2_TOKEN *Token\r
+ )\r
+{\r
+ EFI_STATUS Status;\r
+ UINT32 BlockSize;\r
+ NVME_CONTROLLER_PRIVATE_DATA *Private;\r
+ NVME_BLKIO2_REQUEST *BlkIo2Req;\r
+ UINT32 MaxTransferBlocks;\r
+ UINTN OrginalBlocks;\r
+ BOOLEAN IsEmpty;\r
+ EFI_TPL OldTpl;\r
+\r
+ Status = EFI_SUCCESS;\r
+ Private = Device->Controller;\r
+ BlockSize = Device->Media.BlockSize;\r
+ OrginalBlocks = Blocks;\r
+ BlkIo2Req = AllocateZeroPool (sizeof (NVME_BLKIO2_REQUEST));\r
+ if (BlkIo2Req == NULL) {\r
+ return EFI_OUT_OF_RESOURCES;\r
+ }\r
+\r
+ BlkIo2Req->Signature = NVME_BLKIO2_REQUEST_SIGNATURE;\r
+ BlkIo2Req->Token = Token;\r
+\r
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);\r
+ InsertTailList (&Device->AsyncQueue, &BlkIo2Req->Link);\r
+ gBS->RestoreTPL (OldTpl);\r
+\r
+ InitializeListHead (&BlkIo2Req->SubtasksQueue);\r
+\r
+ if (Private->ControllerData->Mdts != 0) {\r
+ MaxTransferBlocks = (1 << (Private->ControllerData->Mdts)) * (1 << (Private->Cap.Mpsmin + 12)) / BlockSize;\r
+ } else {\r
+ MaxTransferBlocks = 1024;\r
+ }\r
+\r
+ while (Blocks > 0) {\r
+ if (Blocks > MaxTransferBlocks) {\r
+ Status = AsyncWriteSectors (\r
+ Device,\r
+ BlkIo2Req,\r
+ (UINT64)(UINTN)Buffer,\r
+ Lba,\r
+ MaxTransferBlocks,\r
+ FALSE\r
+ );\r
+\r
+ Blocks -= MaxTransferBlocks;\r
+ Buffer = (VOID *)(UINTN)((UINT64)(UINTN)Buffer + MaxTransferBlocks * BlockSize);\r
+ Lba += MaxTransferBlocks;\r
+ } else {\r
+ Status = AsyncWriteSectors (\r
+ Device,\r
+ BlkIo2Req,\r
+ (UINT64)(UINTN)Buffer,\r
+ Lba,\r
+ (UINT32)Blocks,\r
+ TRUE\r
+ );\r
+\r
+ Blocks = 0;\r
+ }\r
+\r
+ if (EFI_ERROR(Status)) {\r
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);\r
+ IsEmpty = IsListEmpty (&BlkIo2Req->SubtasksQueue) &&\r
+ (BlkIo2Req->UnsubmittedSubtaskNum == 0);\r
+\r
+ if (IsEmpty) {\r
+ //\r
+ // Remove the BlockIo2 request from the device asynchronous queue.\r
+ //\r
+ RemoveEntryList (&BlkIo2Req->Link);\r
+ FreePool (BlkIo2Req);\r
+ Status = EFI_DEVICE_ERROR;\r
+ } else {\r
+ //\r
+ // There are previous BlockIo2 subtasks still running, EFI_SUCCESS\r
+ // should be returned to make sure that the caller does not free\r
+ // resources still using by these requests.\r
+ //\r
+ Status = EFI_SUCCESS;\r
+ Token->TransactionStatus = EFI_DEVICE_ERROR;\r
+ BlkIo2Req->LastSubtaskSubmitted = TRUE;\r
+ }\r
+\r
+ gBS->RestoreTPL (OldTpl);\r
+\r
+ break;\r
+ }\r
+ }\r
+\r
+ DEBUG ((EFI_D_VERBOSE, "%a: Lba = 0x%08Lx, Original = 0x%08Lx, "\r
+ "Remaining = 0x%08Lx, BlockSize = 0x%x, Status = %r\n", __FUNCTION__, Lba,\r
+ (UINT64)OrginalBlocks, (UINT64)Blocks, BlockSize, Status));\r
+\r
+ return Status;\r
+}\r