2 NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows
3 NVM Express specification.
5 (C) Copyright 2014 Hewlett-Packard Development Company, L.P.<BR>
6 Copyright (c) 2013 - 2018, Intel Corporation. All rights reserved.<BR>
7 SPDX-License-Identifier: BSD-2-Clause-Patent
11 #include "NvmExpress.h"
14 Dump the execution status from a given completion queue entry.
16 @param[in] Cq A pointer to the NVME_CQ item.
24 DEBUG ((DEBUG_VERBOSE
, "Dump NVMe Completion Entry Status from [0x%x]:\n", Cq
));
26 DEBUG ((DEBUG_VERBOSE
, " SQ Identifier : [0x%x], Phase Tag : [%d], Cmd Identifier : [0x%x]\n", Cq
->Sqid
, Cq
->Pt
, Cq
->Cid
));
28 DEBUG ((DEBUG_VERBOSE
, " NVMe Cmd Execution Result - "));
34 DEBUG ((DEBUG_VERBOSE
, "Successful Completion\n"));
37 DEBUG ((DEBUG_VERBOSE
, "Invalid Command Opcode\n"));
40 DEBUG ((DEBUG_VERBOSE
, "Invalid Field in Command\n"));
43 DEBUG ((DEBUG_VERBOSE
, "Command ID Conflict\n"));
46 DEBUG ((DEBUG_VERBOSE
, "Data Transfer Error\n"));
49 DEBUG ((DEBUG_VERBOSE
, "Commands Aborted due to Power Loss Notification\n"));
52 DEBUG ((DEBUG_VERBOSE
, "Internal Device Error\n"));
55 DEBUG ((DEBUG_VERBOSE
, "Command Abort Requested\n"));
58 DEBUG ((DEBUG_VERBOSE
, "Command Aborted due to SQ Deletion\n"));
61 DEBUG ((DEBUG_VERBOSE
, "Command Aborted due to Failed Fused Command\n"));
64 DEBUG ((DEBUG_VERBOSE
, "Command Aborted due to Missing Fused Command\n"));
67 DEBUG ((DEBUG_VERBOSE
, "Invalid Namespace or Format\n"));
70 DEBUG ((DEBUG_VERBOSE
, "Command Sequence Error\n"));
73 DEBUG ((DEBUG_VERBOSE
, "Invalid SGL Last Segment Descriptor\n"));
76 DEBUG ((DEBUG_VERBOSE
, "Invalid Number of SGL Descriptors\n"));
79 DEBUG ((DEBUG_VERBOSE
, "Data SGL Length Invalid\n"));
82 DEBUG ((DEBUG_VERBOSE
, "Metadata SGL Length Invalid\n"));
85 DEBUG ((DEBUG_VERBOSE
, "SGL Descriptor Type Invalid\n"));
88 DEBUG ((DEBUG_VERBOSE
, "LBA Out of Range\n"));
91 DEBUG ((DEBUG_VERBOSE
, "Capacity Exceeded\n"));
94 DEBUG ((DEBUG_VERBOSE
, "Namespace Not Ready\n"));
97 DEBUG ((DEBUG_VERBOSE
, "Reservation Conflict\n"));
105 DEBUG ((DEBUG_VERBOSE
, "Completion Queue Invalid\n"));
108 DEBUG ((DEBUG_VERBOSE
, "Invalid Queue Identifier\n"));
111 DEBUG ((DEBUG_VERBOSE
, "Maximum Queue Size Exceeded\n"));
114 DEBUG ((DEBUG_VERBOSE
, "Abort Command Limit Exceeded\n"));
117 DEBUG ((DEBUG_VERBOSE
, "Asynchronous Event Request Limit Exceeded\n"));
120 DEBUG ((DEBUG_VERBOSE
, "Invalid Firmware Slot\n"));
123 DEBUG ((DEBUG_VERBOSE
, "Invalid Firmware Image\n"));
126 DEBUG ((DEBUG_VERBOSE
, "Invalid Interrupt Vector\n"));
129 DEBUG ((DEBUG_VERBOSE
, "Invalid Log Page\n"));
132 DEBUG ((DEBUG_VERBOSE
, "Invalid Format\n"));
135 DEBUG ((DEBUG_VERBOSE
, "Firmware Application Requires Conventional Reset\n"));
138 DEBUG ((DEBUG_VERBOSE
, "Invalid Queue Deletion\n"));
141 DEBUG ((DEBUG_VERBOSE
, "Feature Identifier Not Saveable\n"));
144 DEBUG ((DEBUG_VERBOSE
, "Feature Not Changeable\n"));
147 DEBUG ((DEBUG_VERBOSE
, "Feature Not Namespace Specific\n"));
150 DEBUG ((DEBUG_VERBOSE
, "Firmware Application Requires NVM Subsystem Reset\n"));
153 DEBUG ((DEBUG_VERBOSE
, "Conflicting Attributes\n"));
156 DEBUG ((DEBUG_VERBOSE
, "Invalid Protection Information\n"));
159 DEBUG ((DEBUG_VERBOSE
, "Attempted Write to Read Only Range\n"));
167 DEBUG ((DEBUG_VERBOSE
, "Write Fault\n"));
170 DEBUG ((DEBUG_VERBOSE
, "Unrecovered Read Error\n"));
173 DEBUG ((DEBUG_VERBOSE
, "End-to-end Guard Check Error\n"));
176 DEBUG ((DEBUG_VERBOSE
, "End-to-end Application Tag Check Error\n"));
179 DEBUG ((DEBUG_VERBOSE
, "End-to-end Reference Tag Check Error\n"));
182 DEBUG ((DEBUG_VERBOSE
, "Compare Failure\n"));
185 DEBUG ((DEBUG_VERBOSE
, "Access Denied\n"));
196 Create PRP lists for data transfer which is larger than 2 memory pages.
197 Note here we calcuate the number of required PRP lists and allocate them at one time.
199 @param[in] PciIo A pointer to the EFI_PCI_IO_PROTOCOL instance.
200 @param[in] PhysicalAddr The physical base address of data buffer.
201 @param[in] Pages The number of pages to be transfered.
202 @param[out] PrpListHost The host base address of PRP lists.
203 @param[in,out] PrpListNo The number of PRP List.
204 @param[out] Mapping The mapping value returned from PciIo.Map().
206 @retval The pointer to the first PRP List of the PRP lists.
211 IN EFI_PCI_IO_PROTOCOL
*PciIo
,
212 IN EFI_PHYSICAL_ADDRESS PhysicalAddr
,
214 OUT VOID
**PrpListHost
,
215 IN OUT UINTN
*PrpListNo
,
224 EFI_PHYSICAL_ADDRESS PrpListPhyAddr
;
229 // The number of Prp Entry in a memory page.
231 PrpEntryNo
= EFI_PAGE_SIZE
/ sizeof (UINT64
);
234 // Calculate total PrpList number.
236 *PrpListNo
= (UINTN
)DivU64x64Remainder ((UINT64
)Pages
, (UINT64
)PrpEntryNo
- 1, &Remainder
);
237 if (*PrpListNo
== 0) {
239 } else if ((Remainder
!= 0) && (Remainder
!= 1)) {
241 } else if (Remainder
== 1) {
242 Remainder
= PrpEntryNo
;
243 } else if (Remainder
== 0) {
244 Remainder
= PrpEntryNo
- 1;
247 Status
= PciIo
->AllocateBuffer (
256 if (EFI_ERROR (Status
)) {
260 Bytes
= EFI_PAGES_TO_SIZE (*PrpListNo
);
261 Status
= PciIo
->Map (
263 EfiPciIoOperationBusMasterCommonBuffer
,
270 if (EFI_ERROR (Status
) || (Bytes
!= EFI_PAGES_TO_SIZE (*PrpListNo
))) {
271 DEBUG ((DEBUG_ERROR
, "NvmeCreatePrpList: create PrpList failure!\n"));
275 // Fill all PRP lists except of last one.
277 ZeroMem (*PrpListHost
, Bytes
);
278 for (PrpListIndex
= 0; PrpListIndex
< *PrpListNo
- 1; ++PrpListIndex
) {
279 PrpListBase
= *(UINT64
*)PrpListHost
+ PrpListIndex
* EFI_PAGE_SIZE
;
281 for (PrpEntryIndex
= 0; PrpEntryIndex
< PrpEntryNo
; ++PrpEntryIndex
) {
282 if (PrpEntryIndex
!= PrpEntryNo
- 1) {
284 // Fill all PRP entries except of last one.
286 *((UINT64
*)(UINTN
)PrpListBase
+ PrpEntryIndex
) = PhysicalAddr
;
287 PhysicalAddr
+= EFI_PAGE_SIZE
;
290 // Fill last PRP entries with next PRP List pointer.
292 *((UINT64
*)(UINTN
)PrpListBase
+ PrpEntryIndex
) = PrpListPhyAddr
+ (PrpListIndex
+ 1) * EFI_PAGE_SIZE
;
297 // Fill last PRP list.
299 PrpListBase
= *(UINT64
*)PrpListHost
+ PrpListIndex
* EFI_PAGE_SIZE
;
300 for (PrpEntryIndex
= 0; PrpEntryIndex
< Remainder
; ++PrpEntryIndex
) {
301 *((UINT64
*)(UINTN
)PrpListBase
+ PrpEntryIndex
) = PhysicalAddr
;
302 PhysicalAddr
+= EFI_PAGE_SIZE
;
305 return (VOID
*)(UINTN
)PrpListPhyAddr
;
308 PciIo
->FreeBuffer (PciIo
, *PrpListNo
, *PrpListHost
);
314 Aborts the asynchronous PassThru requests.
316 @param[in] Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA
319 @retval EFI_SUCCESS The asynchronous PassThru requests have been aborted.
320 @return EFI_DEVICE_ERROR Fail to abort all the asynchronous PassThru requests.
324 AbortAsyncPassThruTasks (
325 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
328 EFI_PCI_IO_PROTOCOL
*PciIo
;
330 LIST_ENTRY
*NextLink
;
331 NVME_BLKIO2_SUBTASK
*Subtask
;
332 NVME_BLKIO2_REQUEST
*BlkIo2Request
;
333 NVME_PASS_THRU_ASYNC_REQ
*AsyncRequest
;
334 EFI_BLOCK_IO2_TOKEN
*Token
;
338 PciIo
= Private
->PciIo
;
339 OldTpl
= gBS
->RaiseTPL (TPL_NOTIFY
);
342 // Cancel the unsubmitted subtasks.
344 for (Link
= GetFirstNode (&Private
->UnsubmittedSubtasks
);
345 !IsNull (&Private
->UnsubmittedSubtasks
, Link
);
347 NextLink
= GetNextNode (&Private
->UnsubmittedSubtasks
, Link
);
348 Subtask
= NVME_BLKIO2_SUBTASK_FROM_LINK (Link
);
349 BlkIo2Request
= Subtask
->BlockIo2Request
;
350 Token
= BlkIo2Request
->Token
;
352 BlkIo2Request
->UnsubmittedSubtaskNum
--;
353 if (Subtask
->IsLast
) {
354 BlkIo2Request
->LastSubtaskSubmitted
= TRUE
;
356 Token
->TransactionStatus
= EFI_ABORTED
;
358 RemoveEntryList (Link
);
359 InsertTailList (&BlkIo2Request
->SubtasksQueue
, Link
);
360 gBS
->SignalEvent (Subtask
->Event
);
364 // Cleanup the resources for the asynchronous PassThru requests.
366 for (Link
= GetFirstNode (&Private
->AsyncPassThruQueue
);
367 !IsNull (&Private
->AsyncPassThruQueue
, Link
);
369 NextLink
= GetNextNode (&Private
->AsyncPassThruQueue
, Link
);
370 AsyncRequest
= NVME_PASS_THRU_ASYNC_REQ_FROM_THIS (Link
);
372 if (AsyncRequest
->MapData
!= NULL
) {
373 PciIo
->Unmap (PciIo
, AsyncRequest
->MapData
);
375 if (AsyncRequest
->MapMeta
!= NULL
) {
376 PciIo
->Unmap (PciIo
, AsyncRequest
->MapMeta
);
378 if (AsyncRequest
->MapPrpList
!= NULL
) {
379 PciIo
->Unmap (PciIo
, AsyncRequest
->MapPrpList
);
381 if (AsyncRequest
->PrpListHost
!= NULL
) {
384 AsyncRequest
->PrpListNo
,
385 AsyncRequest
->PrpListHost
389 RemoveEntryList (Link
);
390 gBS
->SignalEvent (AsyncRequest
->CallerEvent
);
391 FreePool (AsyncRequest
);
394 if (IsListEmpty (&Private
->AsyncPassThruQueue
) &&
395 IsListEmpty (&Private
->UnsubmittedSubtasks
)) {
396 Status
= EFI_SUCCESS
;
398 Status
= EFI_DEVICE_ERROR
;
401 gBS
->RestoreTPL (OldTpl
);
408 Sends an NVM Express Command Packet to an NVM Express controller or namespace. This function supports
409 both blocking I/O and non-blocking I/O. The blocking I/O functionality is required, and the non-blocking
410 I/O functionality is optional.
413 @param[in] This A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
414 @param[in] NamespaceId A 32 bit namespace ID as defined in the NVMe specification to which the NVM Express Command
415 Packet will be sent. A value of 0 denotes the NVM Express controller, a value of all 0xFF's
416 (all bytes are 0xFF) in the namespace ID specifies that the command packet should be sent to
417 all valid namespaces.
418 @param[in,out] Packet A pointer to the NVM Express Command Packet.
419 @param[in] Event If non-blocking I/O is not supported then Event is ignored, and blocking I/O is performed.
420 If Event is NULL, then blocking I/O is performed. If Event is not NULL and non-blocking I/O
421 is supported, then non-blocking I/O is performed, and Event will be signaled when the NVM
422 Express Command Packet completes.
424 @retval EFI_SUCCESS The NVM Express Command Packet was sent by the host. TransferLength bytes were transferred
425 to, or from DataBuffer.
426 @retval EFI_BAD_BUFFER_SIZE The NVM Express Command Packet was not executed. The number of bytes that could be transferred
427 is returned in TransferLength.
428 @retval EFI_NOT_READY The NVM Express Command Packet could not be sent because the controller is not ready. The caller
429 may retry again later.
430 @retval EFI_DEVICE_ERROR A device error occurred while attempting to send the NVM Express Command Packet.
431 @retval EFI_INVALID_PARAMETER NamespaceId or the contents of EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET are invalid. The NVM
432 Express Command Packet was not sent, so no additional status information is available.
433 @retval EFI_UNSUPPORTED The command described by the NVM Express Command Packet is not supported by the NVM Express
434 controller. The NVM Express Command Packet was not sent so no additional status information
436 @retval EFI_TIMEOUT A timeout occurred while waiting for the NVM Express Command Packet to execute.
442 IN EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL
*This
,
443 IN UINT32 NamespaceId
,
444 IN OUT EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET
*Packet
,
445 IN EFI_EVENT Event OPTIONAL
448 NVME_CONTROLLER_PRIVATE_DATA
*Private
;
450 EFI_STATUS PreviousStatus
;
451 EFI_PCI_IO_PROTOCOL
*PciIo
;
458 EFI_EVENT TimerEvent
;
459 EFI_PCI_IO_PROTOCOL_OPERATION Flag
;
460 EFI_PHYSICAL_ADDRESS PhyAddr
;
472 NVME_PASS_THRU_ASYNC_REQ
*AsyncRequest
;
476 // check the data fields in Packet parameter.
478 if ((This
== NULL
) || (Packet
== NULL
)) {
479 return EFI_INVALID_PARAMETER
;
482 if ((Packet
->NvmeCmd
== NULL
) || (Packet
->NvmeCompletion
== NULL
)) {
483 return EFI_INVALID_PARAMETER
;
486 if (Packet
->QueueType
!= NVME_ADMIN_QUEUE
&& Packet
->QueueType
!= NVME_IO_QUEUE
) {
487 return EFI_INVALID_PARAMETER
;
491 // 'Attributes' with neither EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL nor
492 // EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL set is an illegal
495 Attributes
= This
->Mode
->Attributes
;
496 if ((Attributes
& (EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL
|
497 EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL
)) == 0) {
498 return EFI_INVALID_PARAMETER
;
502 // Buffer alignment check for TransferBuffer & MetadataBuffer.
504 IoAlign
= This
->Mode
->IoAlign
;
505 if (IoAlign
> 0 && (((UINTN
) Packet
->TransferBuffer
& (IoAlign
- 1)) != 0)) {
506 return EFI_INVALID_PARAMETER
;
509 if (IoAlign
> 0 && (((UINTN
) Packet
->MetadataBuffer
& (IoAlign
- 1)) != 0)) {
510 return EFI_INVALID_PARAMETER
;
513 Private
= NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This
);
516 // Check NamespaceId is valid or not.
518 if ((NamespaceId
> Private
->ControllerData
->Nn
) &&
519 (NamespaceId
!= (UINT32
) -1)) {
520 return EFI_INVALID_PARAMETER
;
524 // Check whether TransferLength exceeds the maximum data transfer size.
526 if (Private
->ControllerData
->Mdts
!= 0) {
527 MaxTransLen
= (1 << (Private
->ControllerData
->Mdts
)) *
528 (1 << (Private
->Cap
.Mpsmin
+ 12));
529 if (Packet
->TransferLength
> MaxTransLen
) {
530 Packet
->TransferLength
= MaxTransLen
;
531 return EFI_BAD_BUFFER_SIZE
;
535 PciIo
= Private
->PciIo
;
543 Status
= EFI_SUCCESS
;
544 QueueSize
= MIN (NVME_ASYNC_CSQ_SIZE
, Private
->Cap
.Mqes
) + 1;
546 if (Packet
->QueueType
== NVME_ADMIN_QUEUE
) {
555 // Submission queue full check.
557 if ((Private
->SqTdbl
[QueueId
].Sqt
+ 1) % QueueSize
==
558 Private
->AsyncSqHead
) {
559 return EFI_NOT_READY
;
563 Sq
= Private
->SqBuffer
[QueueId
] + Private
->SqTdbl
[QueueId
].Sqt
;
564 Cq
= Private
->CqBuffer
[QueueId
] + Private
->CqHdbl
[QueueId
].Cqh
;
566 if (Packet
->NvmeCmd
->Nsid
!= NamespaceId
) {
567 return EFI_INVALID_PARAMETER
;
570 ZeroMem (Sq
, sizeof (NVME_SQ
));
571 Sq
->Opc
= (UINT8
)Packet
->NvmeCmd
->Cdw0
.Opcode
;
572 Sq
->Fuse
= (UINT8
)Packet
->NvmeCmd
->Cdw0
.FusedOperation
;
573 Sq
->Cid
= Private
->Cid
[QueueId
]++;
574 Sq
->Nsid
= Packet
->NvmeCmd
->Nsid
;
577 // Currently we only support PRP for data transfer, SGL is NOT supported.
579 ASSERT (Sq
->Psdt
== 0);
581 DEBUG ((DEBUG_ERROR
, "NvmExpressPassThru: doesn't support SGL mechanism\n"));
582 return EFI_UNSUPPORTED
;
585 Sq
->Prp
[0] = (UINT64
)(UINTN
)Packet
->TransferBuffer
;
586 if ((Packet
->QueueType
== NVME_ADMIN_QUEUE
) &&
587 ((Sq
->Opc
== NVME_ADMIN_CRIOCQ_CMD
) || (Sq
->Opc
== NVME_ADMIN_CRIOSQ_CMD
))) {
589 // Currently, we only use the IO Completion/Submission queues created internally
590 // by this driver during controller initialization. Any other IO queues created
591 // will not be consumed here. The value is little to accept external IO queue
592 // creation requests, so here we will return EFI_UNSUPPORTED for external IO
593 // queue creation request.
595 if (!Private
->CreateIoQueue
) {
596 DEBUG ((DEBUG_ERROR
, "NvmExpressPassThru: Does not support external IO queues creation request.\n"));
597 return EFI_UNSUPPORTED
;
599 } else if ((Sq
->Opc
& (BIT0
| BIT1
)) != 0) {
601 // If the NVMe cmd has data in or out, then mapping the user buffer to the PCI controller specific addresses.
603 if (((Packet
->TransferLength
!= 0) && (Packet
->TransferBuffer
== NULL
)) ||
604 ((Packet
->TransferLength
== 0) && (Packet
->TransferBuffer
!= NULL
))) {
605 return EFI_INVALID_PARAMETER
;
608 if ((Sq
->Opc
& BIT0
) != 0) {
609 Flag
= EfiPciIoOperationBusMasterRead
;
611 Flag
= EfiPciIoOperationBusMasterWrite
;
614 if ((Packet
->TransferLength
!= 0) && (Packet
->TransferBuffer
!= NULL
)) {
615 MapLength
= Packet
->TransferLength
;
616 Status
= PciIo
->Map (
619 Packet
->TransferBuffer
,
624 if (EFI_ERROR (Status
) || (Packet
->TransferLength
!= MapLength
)) {
625 return EFI_OUT_OF_RESOURCES
;
628 Sq
->Prp
[0] = PhyAddr
;
632 if((Packet
->MetadataLength
!= 0) && (Packet
->MetadataBuffer
!= NULL
)) {
633 MapLength
= Packet
->MetadataLength
;
634 Status
= PciIo
->Map (
637 Packet
->MetadataBuffer
,
642 if (EFI_ERROR (Status
) || (Packet
->MetadataLength
!= MapLength
)) {
648 return EFI_OUT_OF_RESOURCES
;
654 // If the buffer size spans more than two memory pages (page size as defined in CC.Mps),
655 // then build a PRP list in the second PRP submission queue entry.
657 Offset
= ((UINT16
)Sq
->Prp
[0]) & (EFI_PAGE_SIZE
- 1);
658 Bytes
= Packet
->TransferLength
;
660 if ((Offset
+ Bytes
) > (EFI_PAGE_SIZE
* 2)) {
662 // Create PrpList for remaining data buffer.
664 PhyAddr
= (Sq
->Prp
[0] + EFI_PAGE_SIZE
) & ~(EFI_PAGE_SIZE
- 1);
665 Prp
= NvmeCreatePrpList (PciIo
, PhyAddr
, EFI_SIZE_TO_PAGES(Offset
+ Bytes
) - 1, &PrpListHost
, &PrpListNo
, &MapPrpList
);
667 Status
= EFI_OUT_OF_RESOURCES
;
671 Sq
->Prp
[1] = (UINT64
)(UINTN
)Prp
;
672 } else if ((Offset
+ Bytes
) > EFI_PAGE_SIZE
) {
673 Sq
->Prp
[1] = (Sq
->Prp
[0] + EFI_PAGE_SIZE
) & ~(EFI_PAGE_SIZE
- 1);
676 if(Packet
->NvmeCmd
->Flags
& CDW2_VALID
) {
677 Sq
->Rsvd2
= (UINT64
)Packet
->NvmeCmd
->Cdw2
;
679 if(Packet
->NvmeCmd
->Flags
& CDW3_VALID
) {
680 Sq
->Rsvd2
|= LShiftU64 ((UINT64
)Packet
->NvmeCmd
->Cdw3
, 32);
682 if(Packet
->NvmeCmd
->Flags
& CDW10_VALID
) {
683 Sq
->Payload
.Raw
.Cdw10
= Packet
->NvmeCmd
->Cdw10
;
685 if(Packet
->NvmeCmd
->Flags
& CDW11_VALID
) {
686 Sq
->Payload
.Raw
.Cdw11
= Packet
->NvmeCmd
->Cdw11
;
688 if(Packet
->NvmeCmd
->Flags
& CDW12_VALID
) {
689 Sq
->Payload
.Raw
.Cdw12
= Packet
->NvmeCmd
->Cdw12
;
691 if(Packet
->NvmeCmd
->Flags
& CDW13_VALID
) {
692 Sq
->Payload
.Raw
.Cdw13
= Packet
->NvmeCmd
->Cdw13
;
694 if(Packet
->NvmeCmd
->Flags
& CDW14_VALID
) {
695 Sq
->Payload
.Raw
.Cdw14
= Packet
->NvmeCmd
->Cdw14
;
697 if(Packet
->NvmeCmd
->Flags
& CDW15_VALID
) {
698 Sq
->Payload
.Raw
.Cdw15
= Packet
->NvmeCmd
->Cdw15
;
702 // Ring the submission queue doorbell.
704 if ((Event
!= NULL
) && (QueueId
!= 0)) {
705 Private
->SqTdbl
[QueueId
].Sqt
=
706 (Private
->SqTdbl
[QueueId
].Sqt
+ 1) % QueueSize
;
708 Private
->SqTdbl
[QueueId
].Sqt
^= 1;
710 Data
= ReadUnaligned32 ((UINT32
*)&Private
->SqTdbl
[QueueId
]);
711 Status
= PciIo
->Mem
.Write (
715 NVME_SQTDBL_OFFSET(QueueId
, Private
->Cap
.Dstrd
),
720 if (EFI_ERROR (Status
)) {
725 // For non-blocking requests, return directly if the command is placed
726 // in the submission queue.
728 if ((Event
!= NULL
) && (QueueId
!= 0)) {
729 AsyncRequest
= AllocateZeroPool (sizeof (NVME_PASS_THRU_ASYNC_REQ
));
730 if (AsyncRequest
== NULL
) {
731 Status
= EFI_DEVICE_ERROR
;
735 AsyncRequest
->Signature
= NVME_PASS_THRU_ASYNC_REQ_SIG
;
736 AsyncRequest
->Packet
= Packet
;
737 AsyncRequest
->CommandId
= Sq
->Cid
;
738 AsyncRequest
->CallerEvent
= Event
;
739 AsyncRequest
->MapData
= MapData
;
740 AsyncRequest
->MapMeta
= MapMeta
;
741 AsyncRequest
->MapPrpList
= MapPrpList
;
742 AsyncRequest
->PrpListNo
= PrpListNo
;
743 AsyncRequest
->PrpListHost
= PrpListHost
;
745 OldTpl
= gBS
->RaiseTPL (TPL_NOTIFY
);
746 InsertTailList (&Private
->AsyncPassThruQueue
, &AsyncRequest
->Link
);
747 gBS
->RestoreTPL (OldTpl
);
752 Status
= gBS
->CreateEvent (
759 if (EFI_ERROR (Status
)) {
763 Status
= gBS
->SetTimer(TimerEvent
, TimerRelative
, Packet
->CommandTimeout
);
765 if (EFI_ERROR(Status
)) {
770 // Wait for completion queue to get filled in.
772 Status
= EFI_TIMEOUT
;
773 while (EFI_ERROR (gBS
->CheckEvent (TimerEvent
))) {
774 if (Cq
->Pt
!= Private
->Pt
[QueueId
]) {
775 Status
= EFI_SUCCESS
;
781 // Check the NVMe cmd execution result
783 if (Status
!= EFI_TIMEOUT
) {
784 if ((Cq
->Sct
== 0) && (Cq
->Sc
== 0)) {
785 Status
= EFI_SUCCESS
;
787 Status
= EFI_DEVICE_ERROR
;
789 // Dump every completion entry status for debugging.
796 // Copy the Respose Queue entry for this command to the callers response buffer
798 CopyMem(Packet
->NvmeCompletion
, Cq
, sizeof(EFI_NVM_EXPRESS_COMPLETION
));
801 // Timeout occurs for an NVMe command. Reset the controller to abort the
802 // outstanding commands.
804 DEBUG ((DEBUG_ERROR
, "NvmExpressPassThru: Timeout occurs for an NVMe command.\n"));
807 // Disable the timer to trigger the process of async transfers temporarily.
809 Status
= gBS
->SetTimer (Private
->TimerEvent
, TimerCancel
, 0);
810 if (EFI_ERROR (Status
)) {
815 // Reset the NVMe controller.
817 Status
= NvmeControllerInit (Private
);
818 if (!EFI_ERROR (Status
)) {
819 Status
= AbortAsyncPassThruTasks (Private
);
820 if (!EFI_ERROR (Status
)) {
822 // Re-enable the timer to trigger the process of async transfers.
824 Status
= gBS
->SetTimer (Private
->TimerEvent
, TimerPeriodic
, NVME_HC_ASYNC_TIMER
);
825 if (!EFI_ERROR (Status
)) {
827 // Return EFI_TIMEOUT to indicate a timeout occurs for NVMe PassThru command.
829 Status
= EFI_TIMEOUT
;
833 Status
= EFI_DEVICE_ERROR
;
839 if ((Private
->CqHdbl
[QueueId
].Cqh
^= 1) == 0) {
840 Private
->Pt
[QueueId
] ^= 1;
843 Data
= ReadUnaligned32 ((UINT32
*)&Private
->CqHdbl
[QueueId
]);
844 PreviousStatus
= Status
;
845 Status
= PciIo
->Mem
.Write (
849 NVME_CQHDBL_OFFSET(QueueId
, Private
->Cap
.Dstrd
),
853 // The return status of PciIo->Mem.Write should not override
854 // previous status if previous status contains error.
855 Status
= EFI_ERROR (PreviousStatus
) ? PreviousStatus
: Status
;
858 // For now, the code does not support the non-blocking feature for admin queue.
859 // If Event is not NULL for admin queue, signal the caller's event here.
862 ASSERT (QueueId
== 0);
863 gBS
->SignalEvent (Event
);
867 if (MapData
!= NULL
) {
874 if (MapMeta
!= NULL
) {
881 if (MapPrpList
!= NULL
) {
889 PciIo
->FreeBuffer (PciIo
, PrpListNo
, PrpListHost
);
892 if (TimerEvent
!= NULL
) {
893 gBS
->CloseEvent (TimerEvent
);
899 Used to retrieve the next namespace ID for this NVM Express controller.
901 The EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL.GetNextNamespace() function retrieves the next valid
902 namespace ID on this NVM Express controller.
904 If on input the value pointed to by NamespaceId is 0xFFFFFFFF, then the first valid namespace
905 ID defined on the NVM Express controller is returned in the location pointed to by NamespaceId
906 and a status of EFI_SUCCESS is returned.
908 If on input the value pointed to by NamespaceId is an invalid namespace ID other than 0xFFFFFFFF,
909 then EFI_INVALID_PARAMETER is returned.
911 If on input the value pointed to by NamespaceId is a valid namespace ID, then the next valid
912 namespace ID on the NVM Express controller is returned in the location pointed to by NamespaceId,
913 and EFI_SUCCESS is returned.
915 If the value pointed to by NamespaceId is the namespace ID of the last namespace on the NVM
916 Express controller, then EFI_NOT_FOUND is returned.
918 @param[in] This A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
919 @param[in,out] NamespaceId On input, a pointer to a legal NamespaceId for an NVM Express
920 namespace present on the NVM Express controller. On output, a
921 pointer to the next NamespaceId of an NVM Express namespace on
922 an NVM Express controller. An input value of 0xFFFFFFFF retrieves
923 the first NamespaceId for an NVM Express namespace present on an
924 NVM Express controller.
926 @retval EFI_SUCCESS The Namespace ID of the next Namespace was returned.
927 @retval EFI_NOT_FOUND There are no more namespaces defined on this controller.
928 @retval EFI_INVALID_PARAMETER NamespaceId is an invalid value other than 0xFFFFFFFF.
933 NvmExpressGetNextNamespace (
934 IN EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL
*This
,
935 IN OUT UINT32
*NamespaceId
938 NVME_CONTROLLER_PRIVATE_DATA
*Private
;
939 NVME_ADMIN_NAMESPACE_DATA
*NamespaceData
;
940 UINT32 NextNamespaceId
;
943 if ((This
== NULL
) || (NamespaceId
== NULL
)) {
944 return EFI_INVALID_PARAMETER
;
947 NamespaceData
= NULL
;
948 Status
= EFI_NOT_FOUND
;
950 Private
= NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This
);
952 // If the NamespaceId input value is 0xFFFFFFFF, then get the first valid namespace ID
954 if (*NamespaceId
== 0xFFFFFFFF) {
956 // Start with the first namespace ID
960 // Allocate buffer for Identify Namespace data.
962 NamespaceData
= (NVME_ADMIN_NAMESPACE_DATA
*)AllocateZeroPool (sizeof (NVME_ADMIN_NAMESPACE_DATA
));
964 if (NamespaceData
== NULL
) {
965 return EFI_NOT_FOUND
;
968 Status
= NvmeIdentifyNamespace (Private
, NextNamespaceId
, NamespaceData
);
969 if (EFI_ERROR(Status
)) {
973 *NamespaceId
= NextNamespaceId
;
975 if (*NamespaceId
> Private
->ControllerData
->Nn
) {
976 return EFI_INVALID_PARAMETER
;
979 NextNamespaceId
= *NamespaceId
+ 1;
980 if (NextNamespaceId
> Private
->ControllerData
->Nn
) {
981 return EFI_NOT_FOUND
;
985 // Allocate buffer for Identify Namespace data.
987 NamespaceData
= (NVME_ADMIN_NAMESPACE_DATA
*)AllocateZeroPool (sizeof (NVME_ADMIN_NAMESPACE_DATA
));
988 if (NamespaceData
== NULL
) {
989 return EFI_NOT_FOUND
;
992 Status
= NvmeIdentifyNamespace (Private
, NextNamespaceId
, NamespaceData
);
993 if (EFI_ERROR(Status
)) {
997 *NamespaceId
= NextNamespaceId
;
1001 if (NamespaceData
!= NULL
) {
1002 FreePool(NamespaceData
);
1009 Used to translate a device path node to a namespace ID.
1011 The EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL.GetNamespace() function determines the namespace ID associated with the
1012 namespace described by DevicePath.
1014 If DevicePath is a device path node type that the NVM Express Pass Thru driver supports, then the NVM Express
1015 Pass Thru driver will attempt to translate the contents DevicePath into a namespace ID.
1017 If this translation is successful, then that namespace ID is returned in NamespaceId, and EFI_SUCCESS is returned
1019 @param[in] This A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
1020 @param[in] DevicePath A pointer to the device path node that describes an NVM Express namespace on
1021 the NVM Express controller.
1022 @param[out] NamespaceId The NVM Express namespace ID contained in the device path node.
1024 @retval EFI_SUCCESS DevicePath was successfully translated to NamespaceId.
1025 @retval EFI_INVALID_PARAMETER If DevicePath or NamespaceId are NULL, then EFI_INVALID_PARAMETER is returned.
1026 @retval EFI_UNSUPPORTED If DevicePath is not a device path node type that the NVM Express Pass Thru driver
1027 supports, then EFI_UNSUPPORTED is returned.
1028 @retval EFI_NOT_FOUND If DevicePath is a device path node type that the NVM Express Pass Thru driver
1029 supports, but there is not a valid translation from DevicePath to a namespace ID,
1030 then EFI_NOT_FOUND is returned.
1034 NvmExpressGetNamespace (
1035 IN EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL
*This
,
1036 IN EFI_DEVICE_PATH_PROTOCOL
*DevicePath
,
1037 OUT UINT32
*NamespaceId
1040 NVME_NAMESPACE_DEVICE_PATH
*Node
;
1041 NVME_CONTROLLER_PRIVATE_DATA
*Private
;
1043 if ((This
== NULL
) || (DevicePath
== NULL
) || (NamespaceId
== NULL
)) {
1044 return EFI_INVALID_PARAMETER
;
1047 if (DevicePath
->Type
!= MESSAGING_DEVICE_PATH
) {
1048 return EFI_UNSUPPORTED
;
1051 Node
= (NVME_NAMESPACE_DEVICE_PATH
*)DevicePath
;
1052 Private
= NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This
);
1054 if (DevicePath
->SubType
== MSG_NVME_NAMESPACE_DP
) {
1055 if (DevicePathNodeLength(DevicePath
) != sizeof(NVME_NAMESPACE_DEVICE_PATH
)) {
1056 return EFI_NOT_FOUND
;
1060 // Check NamespaceId in the device path node is valid or not.
1062 if ((Node
->NamespaceId
== 0) ||
1063 (Node
->NamespaceId
> Private
->ControllerData
->Nn
)) {
1064 return EFI_NOT_FOUND
;
1067 *NamespaceId
= Node
->NamespaceId
;
1071 return EFI_UNSUPPORTED
;
1076 Used to allocate and build a device path node for an NVM Express namespace on an NVM Express controller.
1078 The EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL.BuildDevicePath() function allocates and builds a single device
1079 path node for the NVM Express namespace specified by NamespaceId.
1081 If the NamespaceId is not valid, then EFI_NOT_FOUND is returned.
1083 If DevicePath is NULL, then EFI_INVALID_PARAMETER is returned.
1085 If there are not enough resources to allocate the device path node, then EFI_OUT_OF_RESOURCES is returned.
1087 Otherwise, DevicePath is allocated with the boot service AllocatePool(), the contents of DevicePath are
1088 initialized to describe the NVM Express namespace specified by NamespaceId, and EFI_SUCCESS is returned.
1090 @param[in] This A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
1091 @param[in] NamespaceId The NVM Express namespace ID for which a device path node is to be
1092 allocated and built. Caller must set the NamespaceId to zero if the
1093 device path node will contain a valid UUID.
1094 @param[in,out] DevicePath A pointer to a single device path node that describes the NVM Express
1095 namespace specified by NamespaceId. This function is responsible for
1096 allocating the buffer DevicePath with the boot service AllocatePool().
1097 It is the caller's responsibility to free DevicePath when the caller
1098 is finished with DevicePath.
1099 @retval EFI_SUCCESS The device path node that describes the NVM Express namespace specified
1100 by NamespaceId was allocated and returned in DevicePath.
1101 @retval EFI_NOT_FOUND The NamespaceId is not valid.
1102 @retval EFI_INVALID_PARAMETER DevicePath is NULL.
1103 @retval EFI_OUT_OF_RESOURCES There are not enough resources to allocate the DevicePath node.
1108 NvmExpressBuildDevicePath (
1109 IN EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL
*This
,
1110 IN UINT32 NamespaceId
,
1111 IN OUT EFI_DEVICE_PATH_PROTOCOL
**DevicePath
1114 NVME_NAMESPACE_DEVICE_PATH
*Node
;
1115 NVME_CONTROLLER_PRIVATE_DATA
*Private
;
1117 NVME_ADMIN_NAMESPACE_DATA
*NamespaceData
;
1120 // Validate parameters
1122 if ((This
== NULL
) || (DevicePath
== NULL
)) {
1123 return EFI_INVALID_PARAMETER
;
1126 Status
= EFI_SUCCESS
;
1127 Private
= NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This
);
1130 // Check NamespaceId is valid or not.
1132 if ((NamespaceId
== 0) ||
1133 (NamespaceId
> Private
->ControllerData
->Nn
)) {
1134 return EFI_NOT_FOUND
;
1137 Node
= (NVME_NAMESPACE_DEVICE_PATH
*)AllocateZeroPool (sizeof (NVME_NAMESPACE_DEVICE_PATH
));
1139 return EFI_OUT_OF_RESOURCES
;
1142 Node
->Header
.Type
= MESSAGING_DEVICE_PATH
;
1143 Node
->Header
.SubType
= MSG_NVME_NAMESPACE_DP
;
1144 SetDevicePathNodeLength (&Node
->Header
, sizeof (NVME_NAMESPACE_DEVICE_PATH
));
1145 Node
->NamespaceId
= NamespaceId
;
1148 // Allocate a buffer for Identify Namespace data.
1150 NamespaceData
= NULL
;
1151 NamespaceData
= AllocateZeroPool(sizeof (NVME_ADMIN_NAMESPACE_DATA
));
1152 if(NamespaceData
== NULL
) {
1153 Status
= EFI_OUT_OF_RESOURCES
;
1158 // Get UUID from specified Identify Namespace data.
1160 Status
= NvmeIdentifyNamespace (
1163 (VOID
*)NamespaceData
1166 if (EFI_ERROR(Status
)) {
1170 Node
->NamespaceUuid
= NamespaceData
->Eui64
;
1172 *DevicePath
= (EFI_DEVICE_PATH_PROTOCOL
*)Node
;
1175 if(NamespaceData
!= NULL
) {
1176 FreePool (NamespaceData
);
1179 if (EFI_ERROR (Status
)) {