2 NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows
3 NVM Express specification.
5 (C) Copyright 2014 Hewlett-Packard Development Company, L.P.<BR>
6 Copyright (c) 2013 - 2018, Intel Corporation. All rights reserved.<BR>
7 SPDX-License-Identifier: BSD-2-Clause-Patent
11 #include "NvmExpress.h"
14 Dump the execution status from a given completion queue entry.
16 @param[in] Cq A pointer to the NVME_CQ item.
24 DEBUG ((EFI_D_VERBOSE
, "Dump NVMe Completion Entry Status from [0x%x]:\n", Cq
));
26 DEBUG ((EFI_D_VERBOSE
, " SQ Identifier : [0x%x], Phase Tag : [%d], Cmd Identifier : [0x%x]\n", Cq
->Sqid
, Cq
->Pt
, Cq
->Cid
));
28 DEBUG ((EFI_D_VERBOSE
, " NVMe Cmd Execution Result - "));
34 DEBUG ((EFI_D_VERBOSE
, "Successful Completion\n"));
37 DEBUG ((EFI_D_VERBOSE
, "Invalid Command Opcode\n"));
40 DEBUG ((EFI_D_VERBOSE
, "Invalid Field in Command\n"));
43 DEBUG ((EFI_D_VERBOSE
, "Command ID Conflict\n"));
46 DEBUG ((EFI_D_VERBOSE
, "Data Transfer Error\n"));
49 DEBUG ((EFI_D_VERBOSE
, "Commands Aborted due to Power Loss Notification\n"));
52 DEBUG ((EFI_D_VERBOSE
, "Internal Device Error\n"));
55 DEBUG ((EFI_D_VERBOSE
, "Command Abort Requested\n"));
58 DEBUG ((EFI_D_VERBOSE
, "Command Aborted due to SQ Deletion\n"));
61 DEBUG ((EFI_D_VERBOSE
, "Command Aborted due to Failed Fused Command\n"));
64 DEBUG ((EFI_D_VERBOSE
, "Command Aborted due to Missing Fused Command\n"));
67 DEBUG ((EFI_D_VERBOSE
, "Invalid Namespace or Format\n"));
70 DEBUG ((EFI_D_VERBOSE
, "Command Sequence Error\n"));
73 DEBUG ((EFI_D_VERBOSE
, "Invalid SGL Last Segment Descriptor\n"));
76 DEBUG ((EFI_D_VERBOSE
, "Invalid Number of SGL Descriptors\n"));
79 DEBUG ((EFI_D_VERBOSE
, "Data SGL Length Invalid\n"));
82 DEBUG ((EFI_D_VERBOSE
, "Metadata SGL Length Invalid\n"));
85 DEBUG ((EFI_D_VERBOSE
, "SGL Descriptor Type Invalid\n"));
88 DEBUG ((EFI_D_VERBOSE
, "LBA Out of Range\n"));
91 DEBUG ((EFI_D_VERBOSE
, "Capacity Exceeded\n"));
94 DEBUG ((EFI_D_VERBOSE
, "Namespace Not Ready\n"));
97 DEBUG ((EFI_D_VERBOSE
, "Reservation Conflict\n"));
105 DEBUG ((EFI_D_VERBOSE
, "Completion Queue Invalid\n"));
108 DEBUG ((EFI_D_VERBOSE
, "Invalid Queue Identifier\n"));
111 DEBUG ((EFI_D_VERBOSE
, "Maximum Queue Size Exceeded\n"));
114 DEBUG ((EFI_D_VERBOSE
, "Abort Command Limit Exceeded\n"));
117 DEBUG ((EFI_D_VERBOSE
, "Asynchronous Event Request Limit Exceeded\n"));
120 DEBUG ((EFI_D_VERBOSE
, "Invalid Firmware Slot\n"));
123 DEBUG ((EFI_D_VERBOSE
, "Invalid Firmware Image\n"));
126 DEBUG ((EFI_D_VERBOSE
, "Invalid Interrupt Vector\n"));
129 DEBUG ((EFI_D_VERBOSE
, "Invalid Log Page\n"));
132 DEBUG ((EFI_D_VERBOSE
, "Invalid Format\n"));
135 DEBUG ((EFI_D_VERBOSE
, "Firmware Application Requires Conventional Reset\n"));
138 DEBUG ((EFI_D_VERBOSE
, "Invalid Queue Deletion\n"));
141 DEBUG ((EFI_D_VERBOSE
, "Feature Identifier Not Saveable\n"));
144 DEBUG ((EFI_D_VERBOSE
, "Feature Not Changeable\n"));
147 DEBUG ((EFI_D_VERBOSE
, "Feature Not Namespace Specific\n"));
150 DEBUG ((EFI_D_VERBOSE
, "Firmware Application Requires NVM Subsystem Reset\n"));
153 DEBUG ((EFI_D_VERBOSE
, "Conflicting Attributes\n"));
156 DEBUG ((EFI_D_VERBOSE
, "Invalid Protection Information\n"));
159 DEBUG ((EFI_D_VERBOSE
, "Attempted Write to Read Only Range\n"));
167 DEBUG ((EFI_D_VERBOSE
, "Write Fault\n"));
170 DEBUG ((EFI_D_VERBOSE
, "Unrecovered Read Error\n"));
173 DEBUG ((EFI_D_VERBOSE
, "End-to-end Guard Check Error\n"));
176 DEBUG ((EFI_D_VERBOSE
, "End-to-end Application Tag Check Error\n"));
179 DEBUG ((EFI_D_VERBOSE
, "End-to-end Reference Tag Check Error\n"));
182 DEBUG ((EFI_D_VERBOSE
, "Compare Failure\n"));
185 DEBUG ((EFI_D_VERBOSE
, "Access Denied\n"));
196 Create PRP lists for data transfer which is larger than 2 memory pages.
197 Note here we calcuate the number of required PRP lists and allocate them at one time.
199 @param[in] PciIo A pointer to the EFI_PCI_IO_PROTOCOL instance.
200 @param[in] PhysicalAddr The physical base address of data buffer.
201 @param[in] Pages The number of pages to be transfered.
202 @param[out] PrpListHost The host base address of PRP lists.
203 @param[in,out] PrpListNo The number of PRP List.
204 @param[out] Mapping The mapping value returned from PciIo.Map().
206 @retval The pointer to the first PRP List of the PRP lists.
211 IN EFI_PCI_IO_PROTOCOL
*PciIo
,
212 IN EFI_PHYSICAL_ADDRESS PhysicalAddr
,
214 OUT VOID
**PrpListHost
,
215 IN OUT UINTN
*PrpListNo
,
224 EFI_PHYSICAL_ADDRESS PrpListPhyAddr
;
229 // The number of Prp Entry in a memory page.
231 PrpEntryNo
= EFI_PAGE_SIZE
/ sizeof (UINT64
);
234 // Calculate total PrpList number.
236 *PrpListNo
= (UINTN
)DivU64x64Remainder ((UINT64
)Pages
, (UINT64
)PrpEntryNo
- 1, &Remainder
);
237 if (*PrpListNo
== 0) {
239 } else if ((Remainder
!= 0) && (Remainder
!= 1)) {
241 } else if (Remainder
== 1) {
242 Remainder
= PrpEntryNo
;
243 } else if (Remainder
== 0) {
244 Remainder
= PrpEntryNo
- 1;
247 Status
= PciIo
->AllocateBuffer (
256 if (EFI_ERROR (Status
)) {
260 Bytes
= EFI_PAGES_TO_SIZE (*PrpListNo
);
261 Status
= PciIo
->Map (
263 EfiPciIoOperationBusMasterCommonBuffer
,
270 if (EFI_ERROR (Status
) || (Bytes
!= EFI_PAGES_TO_SIZE (*PrpListNo
))) {
271 DEBUG ((EFI_D_ERROR
, "NvmeCreatePrpList: create PrpList failure!\n"));
275 // Fill all PRP lists except of last one.
277 ZeroMem (*PrpListHost
, Bytes
);
278 for (PrpListIndex
= 0; PrpListIndex
< *PrpListNo
- 1; ++PrpListIndex
) {
279 PrpListBase
= *(UINT64
*)PrpListHost
+ PrpListIndex
* EFI_PAGE_SIZE
;
281 for (PrpEntryIndex
= 0; PrpEntryIndex
< PrpEntryNo
; ++PrpEntryIndex
) {
282 if (PrpEntryIndex
!= PrpEntryNo
- 1) {
284 // Fill all PRP entries except of last one.
286 *((UINT64
*)(UINTN
)PrpListBase
+ PrpEntryIndex
) = PhysicalAddr
;
287 PhysicalAddr
+= EFI_PAGE_SIZE
;
290 // Fill last PRP entries with next PRP List pointer.
292 *((UINT64
*)(UINTN
)PrpListBase
+ PrpEntryIndex
) = PrpListPhyAddr
+ (PrpListIndex
+ 1) * EFI_PAGE_SIZE
;
297 // Fill last PRP list.
299 PrpListBase
= *(UINT64
*)PrpListHost
+ PrpListIndex
* EFI_PAGE_SIZE
;
300 for (PrpEntryIndex
= 0; PrpEntryIndex
< Remainder
; ++PrpEntryIndex
) {
301 *((UINT64
*)(UINTN
)PrpListBase
+ PrpEntryIndex
) = PhysicalAddr
;
302 PhysicalAddr
+= EFI_PAGE_SIZE
;
305 return (VOID
*)(UINTN
)PrpListPhyAddr
;
308 PciIo
->FreeBuffer (PciIo
, *PrpListNo
, *PrpListHost
);
314 Aborts the asynchronous PassThru requests.
316 @param[in] Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA
319 @retval EFI_SUCCESS The asynchronous PassThru requests have been aborted.
320 @return EFI_DEVICE_ERROR Fail to abort all the asynchronous PassThru requests.
324 AbortAsyncPassThruTasks (
325 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
328 EFI_PCI_IO_PROTOCOL
*PciIo
;
330 LIST_ENTRY
*NextLink
;
331 NVME_BLKIO2_SUBTASK
*Subtask
;
332 NVME_BLKIO2_REQUEST
*BlkIo2Request
;
333 NVME_PASS_THRU_ASYNC_REQ
*AsyncRequest
;
334 EFI_BLOCK_IO2_TOKEN
*Token
;
338 PciIo
= Private
->PciIo
;
339 OldTpl
= gBS
->RaiseTPL (TPL_NOTIFY
);
342 // Cancel the unsubmitted subtasks.
344 for (Link
= GetFirstNode (&Private
->UnsubmittedSubtasks
);
345 !IsNull (&Private
->UnsubmittedSubtasks
, Link
);
347 NextLink
= GetNextNode (&Private
->UnsubmittedSubtasks
, Link
);
348 Subtask
= NVME_BLKIO2_SUBTASK_FROM_LINK (Link
);
349 BlkIo2Request
= Subtask
->BlockIo2Request
;
350 Token
= BlkIo2Request
->Token
;
352 BlkIo2Request
->UnsubmittedSubtaskNum
--;
353 if (Subtask
->IsLast
) {
354 BlkIo2Request
->LastSubtaskSubmitted
= TRUE
;
356 Token
->TransactionStatus
= EFI_ABORTED
;
358 RemoveEntryList (Link
);
359 InsertTailList (&BlkIo2Request
->SubtasksQueue
, Link
);
360 gBS
->SignalEvent (Subtask
->Event
);
364 // Cleanup the resources for the asynchronous PassThru requests.
366 for (Link
= GetFirstNode (&Private
->AsyncPassThruQueue
);
367 !IsNull (&Private
->AsyncPassThruQueue
, Link
);
369 NextLink
= GetNextNode (&Private
->AsyncPassThruQueue
, Link
);
370 AsyncRequest
= NVME_PASS_THRU_ASYNC_REQ_FROM_THIS (Link
);
372 if (AsyncRequest
->MapData
!= NULL
) {
373 PciIo
->Unmap (PciIo
, AsyncRequest
->MapData
);
375 if (AsyncRequest
->MapMeta
!= NULL
) {
376 PciIo
->Unmap (PciIo
, AsyncRequest
->MapMeta
);
378 if (AsyncRequest
->MapPrpList
!= NULL
) {
379 PciIo
->Unmap (PciIo
, AsyncRequest
->MapPrpList
);
381 if (AsyncRequest
->PrpListHost
!= NULL
) {
384 AsyncRequest
->PrpListNo
,
385 AsyncRequest
->PrpListHost
389 RemoveEntryList (Link
);
390 gBS
->SignalEvent (AsyncRequest
->CallerEvent
);
391 FreePool (AsyncRequest
);
394 if (IsListEmpty (&Private
->AsyncPassThruQueue
) &&
395 IsListEmpty (&Private
->UnsubmittedSubtasks
)) {
396 Status
= EFI_SUCCESS
;
398 Status
= EFI_DEVICE_ERROR
;
401 gBS
->RestoreTPL (OldTpl
);
408 Sends an NVM Express Command Packet to an NVM Express controller or namespace. This function supports
409 both blocking I/O and non-blocking I/O. The blocking I/O functionality is required, and the non-blocking
410 I/O functionality is optional.
413 @param[in] This A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
414 @param[in] NamespaceId A 32 bit namespace ID as defined in the NVMe specification to which the NVM Express Command
415 Packet will be sent. A value of 0 denotes the NVM Express controller, a value of all 0xFF's
416 (all bytes are 0xFF) in the namespace ID specifies that the command packet should be sent to
417 all valid namespaces.
418 @param[in,out] Packet A pointer to the NVM Express Command Packet.
419 @param[in] Event If non-blocking I/O is not supported then Event is ignored, and blocking I/O is performed.
420 If Event is NULL, then blocking I/O is performed. If Event is not NULL and non-blocking I/O
421 is supported, then non-blocking I/O is performed, and Event will be signaled when the NVM
422 Express Command Packet completes.
424 @retval EFI_SUCCESS The NVM Express Command Packet was sent by the host. TransferLength bytes were transferred
425 to, or from DataBuffer.
426 @retval EFI_BAD_BUFFER_SIZE The NVM Express Command Packet was not executed. The number of bytes that could be transferred
427 is returned in TransferLength.
428 @retval EFI_NOT_READY The NVM Express Command Packet could not be sent because the controller is not ready. The caller
429 may retry again later.
430 @retval EFI_DEVICE_ERROR A device error occurred while attempting to send the NVM Express Command Packet.
431 @retval EFI_INVALID_PARAMETER NamespaceId or the contents of EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET are invalid. The NVM
432 Express Command Packet was not sent, so no additional status information is available.
433 @retval EFI_UNSUPPORTED The command described by the NVM Express Command Packet is not supported by the NVM Express
434 controller. The NVM Express Command Packet was not sent so no additional status information
436 @retval EFI_TIMEOUT A timeout occurred while waiting for the NVM Express Command Packet to execute.
442 IN EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL
*This
,
443 IN UINT32 NamespaceId
,
444 IN OUT EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET
*Packet
,
445 IN EFI_EVENT Event OPTIONAL
448 NVME_CONTROLLER_PRIVATE_DATA
*Private
;
450 EFI_STATUS PreviousStatus
;
451 EFI_PCI_IO_PROTOCOL
*PciIo
;
457 EFI_EVENT TimerEvent
;
458 EFI_PCI_IO_PROTOCOL_OPERATION Flag
;
459 EFI_PHYSICAL_ADDRESS PhyAddr
;
471 NVME_PASS_THRU_ASYNC_REQ
*AsyncRequest
;
475 // check the data fields in Packet parameter.
477 if ((This
== NULL
) || (Packet
== NULL
)) {
478 return EFI_INVALID_PARAMETER
;
481 if ((Packet
->NvmeCmd
== NULL
) || (Packet
->NvmeCompletion
== NULL
)) {
482 return EFI_INVALID_PARAMETER
;
485 if (Packet
->QueueType
!= NVME_ADMIN_QUEUE
&& Packet
->QueueType
!= NVME_IO_QUEUE
) {
486 return EFI_INVALID_PARAMETER
;
490 // 'Attributes' with neither EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL nor
491 // EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL set is an illegal
494 Attributes
= This
->Mode
->Attributes
;
495 if ((Attributes
& (EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL
|
496 EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL
)) == 0) {
497 return EFI_INVALID_PARAMETER
;
501 // Buffer alignment check for TransferBuffer & MetadataBuffer.
503 IoAlign
= This
->Mode
->IoAlign
;
504 if (IoAlign
> 0 && (((UINTN
) Packet
->TransferBuffer
& (IoAlign
- 1)) != 0)) {
505 return EFI_INVALID_PARAMETER
;
508 if (IoAlign
> 0 && (((UINTN
) Packet
->MetadataBuffer
& (IoAlign
- 1)) != 0)) {
509 return EFI_INVALID_PARAMETER
;
512 Private
= NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This
);
515 // Check NamespaceId is valid or not.
517 if ((NamespaceId
> Private
->ControllerData
->Nn
) &&
518 (NamespaceId
!= (UINT32
) -1)) {
519 return EFI_INVALID_PARAMETER
;
523 // Check whether TransferLength exceeds the maximum data transfer size.
525 if (Private
->ControllerData
->Mdts
!= 0) {
526 MaxTransLen
= (1 << (Private
->ControllerData
->Mdts
)) *
527 (1 << (Private
->Cap
.Mpsmin
+ 12));
528 if (Packet
->TransferLength
> MaxTransLen
) {
529 Packet
->TransferLength
= MaxTransLen
;
530 return EFI_BAD_BUFFER_SIZE
;
534 PciIo
= Private
->PciIo
;
542 Status
= EFI_SUCCESS
;
544 if (Packet
->QueueType
== NVME_ADMIN_QUEUE
) {
553 // Submission queue full check.
555 if ((Private
->SqTdbl
[QueueId
].Sqt
+ 1) % (NVME_ASYNC_CSQ_SIZE
+ 1) ==
556 Private
->AsyncSqHead
) {
557 return EFI_NOT_READY
;
561 Sq
= Private
->SqBuffer
[QueueId
] + Private
->SqTdbl
[QueueId
].Sqt
;
562 Cq
= Private
->CqBuffer
[QueueId
] + Private
->CqHdbl
[QueueId
].Cqh
;
564 if (Packet
->NvmeCmd
->Nsid
!= NamespaceId
) {
565 return EFI_INVALID_PARAMETER
;
568 ZeroMem (Sq
, sizeof (NVME_SQ
));
569 Sq
->Opc
= (UINT8
)Packet
->NvmeCmd
->Cdw0
.Opcode
;
570 Sq
->Fuse
= (UINT8
)Packet
->NvmeCmd
->Cdw0
.FusedOperation
;
571 Sq
->Cid
= Private
->Cid
[QueueId
]++;
572 Sq
->Nsid
= Packet
->NvmeCmd
->Nsid
;
575 // Currently we only support PRP for data transfer, SGL is NOT supported.
577 ASSERT (Sq
->Psdt
== 0);
579 DEBUG ((EFI_D_ERROR
, "NvmExpressPassThru: doesn't support SGL mechanism\n"));
580 return EFI_UNSUPPORTED
;
583 Sq
->Prp
[0] = (UINT64
)(UINTN
)Packet
->TransferBuffer
;
584 if ((Packet
->QueueType
== NVME_ADMIN_QUEUE
) &&
585 ((Sq
->Opc
== NVME_ADMIN_CRIOCQ_CMD
) || (Sq
->Opc
== NVME_ADMIN_CRIOSQ_CMD
))) {
587 // Currently, we only use the IO Completion/Submission queues created internally
588 // by this driver during controller initialization. Any other IO queues created
589 // will not be consumed here. The value is little to accept external IO queue
590 // creation requests, so here we will return EFI_UNSUPPORTED for external IO
591 // queue creation request.
593 if (!Private
->CreateIoQueue
) {
594 DEBUG ((DEBUG_ERROR
, "NvmExpressPassThru: Does not support external IO queues creation request.\n"));
595 return EFI_UNSUPPORTED
;
597 } else if ((Sq
->Opc
& (BIT0
| BIT1
)) != 0) {
599 // If the NVMe cmd has data in or out, then mapping the user buffer to the PCI controller specific addresses.
601 if (((Packet
->TransferLength
!= 0) && (Packet
->TransferBuffer
== NULL
)) ||
602 ((Packet
->TransferLength
== 0) && (Packet
->TransferBuffer
!= NULL
))) {
603 return EFI_INVALID_PARAMETER
;
606 if ((Sq
->Opc
& BIT0
) != 0) {
607 Flag
= EfiPciIoOperationBusMasterRead
;
609 Flag
= EfiPciIoOperationBusMasterWrite
;
612 if ((Packet
->TransferLength
!= 0) && (Packet
->TransferBuffer
!= NULL
)) {
613 MapLength
= Packet
->TransferLength
;
614 Status
= PciIo
->Map (
617 Packet
->TransferBuffer
,
622 if (EFI_ERROR (Status
) || (Packet
->TransferLength
!= MapLength
)) {
623 return EFI_OUT_OF_RESOURCES
;
626 Sq
->Prp
[0] = PhyAddr
;
630 if((Packet
->MetadataLength
!= 0) && (Packet
->MetadataBuffer
!= NULL
)) {
631 MapLength
= Packet
->MetadataLength
;
632 Status
= PciIo
->Map (
635 Packet
->MetadataBuffer
,
640 if (EFI_ERROR (Status
) || (Packet
->MetadataLength
!= MapLength
)) {
646 return EFI_OUT_OF_RESOURCES
;
652 // If the buffer size spans more than two memory pages (page size as defined in CC.Mps),
653 // then build a PRP list in the second PRP submission queue entry.
655 Offset
= ((UINT16
)Sq
->Prp
[0]) & (EFI_PAGE_SIZE
- 1);
656 Bytes
= Packet
->TransferLength
;
658 if ((Offset
+ Bytes
) > (EFI_PAGE_SIZE
* 2)) {
660 // Create PrpList for remaining data buffer.
662 PhyAddr
= (Sq
->Prp
[0] + EFI_PAGE_SIZE
) & ~(EFI_PAGE_SIZE
- 1);
663 Prp
= NvmeCreatePrpList (PciIo
, PhyAddr
, EFI_SIZE_TO_PAGES(Offset
+ Bytes
) - 1, &PrpListHost
, &PrpListNo
, &MapPrpList
);
665 Status
= EFI_OUT_OF_RESOURCES
;
669 Sq
->Prp
[1] = (UINT64
)(UINTN
)Prp
;
670 } else if ((Offset
+ Bytes
) > EFI_PAGE_SIZE
) {
671 Sq
->Prp
[1] = (Sq
->Prp
[0] + EFI_PAGE_SIZE
) & ~(EFI_PAGE_SIZE
- 1);
674 if(Packet
->NvmeCmd
->Flags
& CDW2_VALID
) {
675 Sq
->Rsvd2
= (UINT64
)Packet
->NvmeCmd
->Cdw2
;
677 if(Packet
->NvmeCmd
->Flags
& CDW3_VALID
) {
678 Sq
->Rsvd2
|= LShiftU64 ((UINT64
)Packet
->NvmeCmd
->Cdw3
, 32);
680 if(Packet
->NvmeCmd
->Flags
& CDW10_VALID
) {
681 Sq
->Payload
.Raw
.Cdw10
= Packet
->NvmeCmd
->Cdw10
;
683 if(Packet
->NvmeCmd
->Flags
& CDW11_VALID
) {
684 Sq
->Payload
.Raw
.Cdw11
= Packet
->NvmeCmd
->Cdw11
;
686 if(Packet
->NvmeCmd
->Flags
& CDW12_VALID
) {
687 Sq
->Payload
.Raw
.Cdw12
= Packet
->NvmeCmd
->Cdw12
;
689 if(Packet
->NvmeCmd
->Flags
& CDW13_VALID
) {
690 Sq
->Payload
.Raw
.Cdw13
= Packet
->NvmeCmd
->Cdw13
;
692 if(Packet
->NvmeCmd
->Flags
& CDW14_VALID
) {
693 Sq
->Payload
.Raw
.Cdw14
= Packet
->NvmeCmd
->Cdw14
;
695 if(Packet
->NvmeCmd
->Flags
& CDW15_VALID
) {
696 Sq
->Payload
.Raw
.Cdw15
= Packet
->NvmeCmd
->Cdw15
;
700 // Ring the submission queue doorbell.
702 if ((Event
!= NULL
) && (QueueId
!= 0)) {
703 Private
->SqTdbl
[QueueId
].Sqt
=
704 (Private
->SqTdbl
[QueueId
].Sqt
+ 1) % (NVME_ASYNC_CSQ_SIZE
+ 1);
706 Private
->SqTdbl
[QueueId
].Sqt
^= 1;
708 Data
= ReadUnaligned32 ((UINT32
*)&Private
->SqTdbl
[QueueId
]);
709 Status
= PciIo
->Mem
.Write (
713 NVME_SQTDBL_OFFSET(QueueId
, Private
->Cap
.Dstrd
),
718 if (EFI_ERROR (Status
)) {
723 // For non-blocking requests, return directly if the command is placed
724 // in the submission queue.
726 if ((Event
!= NULL
) && (QueueId
!= 0)) {
727 AsyncRequest
= AllocateZeroPool (sizeof (NVME_PASS_THRU_ASYNC_REQ
));
728 if (AsyncRequest
== NULL
) {
729 Status
= EFI_DEVICE_ERROR
;
733 AsyncRequest
->Signature
= NVME_PASS_THRU_ASYNC_REQ_SIG
;
734 AsyncRequest
->Packet
= Packet
;
735 AsyncRequest
->CommandId
= Sq
->Cid
;
736 AsyncRequest
->CallerEvent
= Event
;
737 AsyncRequest
->MapData
= MapData
;
738 AsyncRequest
->MapMeta
= MapMeta
;
739 AsyncRequest
->MapPrpList
= MapPrpList
;
740 AsyncRequest
->PrpListNo
= PrpListNo
;
741 AsyncRequest
->PrpListHost
= PrpListHost
;
743 OldTpl
= gBS
->RaiseTPL (TPL_NOTIFY
);
744 InsertTailList (&Private
->AsyncPassThruQueue
, &AsyncRequest
->Link
);
745 gBS
->RestoreTPL (OldTpl
);
750 Status
= gBS
->CreateEvent (
757 if (EFI_ERROR (Status
)) {
761 Status
= gBS
->SetTimer(TimerEvent
, TimerRelative
, Packet
->CommandTimeout
);
763 if (EFI_ERROR(Status
)) {
768 // Wait for completion queue to get filled in.
770 Status
= EFI_TIMEOUT
;
771 while (EFI_ERROR (gBS
->CheckEvent (TimerEvent
))) {
772 if (Cq
->Pt
!= Private
->Pt
[QueueId
]) {
773 Status
= EFI_SUCCESS
;
779 // Check the NVMe cmd execution result
781 if (Status
!= EFI_TIMEOUT
) {
782 if ((Cq
->Sct
== 0) && (Cq
->Sc
== 0)) {
783 Status
= EFI_SUCCESS
;
785 Status
= EFI_DEVICE_ERROR
;
787 // Dump every completion entry status for debugging.
794 // Copy the Respose Queue entry for this command to the callers response buffer
796 CopyMem(Packet
->NvmeCompletion
, Cq
, sizeof(EFI_NVM_EXPRESS_COMPLETION
));
799 // Timeout occurs for an NVMe command. Reset the controller to abort the
800 // outstanding commands.
802 DEBUG ((DEBUG_ERROR
, "NvmExpressPassThru: Timeout occurs for an NVMe command.\n"));
805 // Disable the timer to trigger the process of async transfers temporarily.
807 Status
= gBS
->SetTimer (Private
->TimerEvent
, TimerCancel
, 0);
808 if (EFI_ERROR (Status
)) {
813 // Reset the NVMe controller.
815 Status
= NvmeControllerInit (Private
);
816 if (!EFI_ERROR (Status
)) {
817 Status
= AbortAsyncPassThruTasks (Private
);
818 if (!EFI_ERROR (Status
)) {
820 // Re-enable the timer to trigger the process of async transfers.
822 Status
= gBS
->SetTimer (Private
->TimerEvent
, TimerPeriodic
, NVME_HC_ASYNC_TIMER
);
823 if (!EFI_ERROR (Status
)) {
825 // Return EFI_TIMEOUT to indicate a timeout occurs for NVMe PassThru command.
827 Status
= EFI_TIMEOUT
;
831 Status
= EFI_DEVICE_ERROR
;
837 if ((Private
->CqHdbl
[QueueId
].Cqh
^= 1) == 0) {
838 Private
->Pt
[QueueId
] ^= 1;
841 Data
= ReadUnaligned32 ((UINT32
*)&Private
->CqHdbl
[QueueId
]);
842 PreviousStatus
= Status
;
843 Status
= PciIo
->Mem
.Write (
847 NVME_CQHDBL_OFFSET(QueueId
, Private
->Cap
.Dstrd
),
851 // The return status of PciIo->Mem.Write should not override
852 // previous status if previous status contains error.
853 Status
= EFI_ERROR (PreviousStatus
) ? PreviousStatus
: Status
;
856 // For now, the code does not support the non-blocking feature for admin queue.
857 // If Event is not NULL for admin queue, signal the caller's event here.
860 ASSERT (QueueId
== 0);
861 gBS
->SignalEvent (Event
);
865 if (MapData
!= NULL
) {
872 if (MapMeta
!= NULL
) {
879 if (MapPrpList
!= NULL
) {
887 PciIo
->FreeBuffer (PciIo
, PrpListNo
, PrpListHost
);
890 if (TimerEvent
!= NULL
) {
891 gBS
->CloseEvent (TimerEvent
);
897 Used to retrieve the next namespace ID for this NVM Express controller.
899 The EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL.GetNextNamespace() function retrieves the next valid
900 namespace ID on this NVM Express controller.
902 If on input the value pointed to by NamespaceId is 0xFFFFFFFF, then the first valid namespace
903 ID defined on the NVM Express controller is returned in the location pointed to by NamespaceId
904 and a status of EFI_SUCCESS is returned.
906 If on input the value pointed to by NamespaceId is an invalid namespace ID other than 0xFFFFFFFF,
907 then EFI_INVALID_PARAMETER is returned.
909 If on input the value pointed to by NamespaceId is a valid namespace ID, then the next valid
910 namespace ID on the NVM Express controller is returned in the location pointed to by NamespaceId,
911 and EFI_SUCCESS is returned.
913 If the value pointed to by NamespaceId is the namespace ID of the last namespace on the NVM
914 Express controller, then EFI_NOT_FOUND is returned.
916 @param[in] This A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
917 @param[in,out] NamespaceId On input, a pointer to a legal NamespaceId for an NVM Express
918 namespace present on the NVM Express controller. On output, a
919 pointer to the next NamespaceId of an NVM Express namespace on
920 an NVM Express controller. An input value of 0xFFFFFFFF retrieves
921 the first NamespaceId for an NVM Express namespace present on an
922 NVM Express controller.
924 @retval EFI_SUCCESS The Namespace ID of the next Namespace was returned.
925 @retval EFI_NOT_FOUND There are no more namespaces defined on this controller.
926 @retval EFI_INVALID_PARAMETER NamespaceId is an invalid value other than 0xFFFFFFFF.
931 NvmExpressGetNextNamespace (
932 IN EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL
*This
,
933 IN OUT UINT32
*NamespaceId
936 NVME_CONTROLLER_PRIVATE_DATA
*Private
;
937 NVME_ADMIN_NAMESPACE_DATA
*NamespaceData
;
938 UINT32 NextNamespaceId
;
941 if ((This
== NULL
) || (NamespaceId
== NULL
)) {
942 return EFI_INVALID_PARAMETER
;
945 NamespaceData
= NULL
;
946 Status
= EFI_NOT_FOUND
;
948 Private
= NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This
);
950 // If the NamespaceId input value is 0xFFFFFFFF, then get the first valid namespace ID
952 if (*NamespaceId
== 0xFFFFFFFF) {
954 // Start with the first namespace ID
958 // Allocate buffer for Identify Namespace data.
960 NamespaceData
= (NVME_ADMIN_NAMESPACE_DATA
*)AllocateZeroPool (sizeof (NVME_ADMIN_NAMESPACE_DATA
));
962 if (NamespaceData
== NULL
) {
963 return EFI_NOT_FOUND
;
966 Status
= NvmeIdentifyNamespace (Private
, NextNamespaceId
, NamespaceData
);
967 if (EFI_ERROR(Status
)) {
971 *NamespaceId
= NextNamespaceId
;
973 if (*NamespaceId
> Private
->ControllerData
->Nn
) {
974 return EFI_INVALID_PARAMETER
;
977 NextNamespaceId
= *NamespaceId
+ 1;
978 if (NextNamespaceId
> Private
->ControllerData
->Nn
) {
979 return EFI_NOT_FOUND
;
983 // Allocate buffer for Identify Namespace data.
985 NamespaceData
= (NVME_ADMIN_NAMESPACE_DATA
*)AllocateZeroPool (sizeof (NVME_ADMIN_NAMESPACE_DATA
));
986 if (NamespaceData
== NULL
) {
987 return EFI_NOT_FOUND
;
990 Status
= NvmeIdentifyNamespace (Private
, NextNamespaceId
, NamespaceData
);
991 if (EFI_ERROR(Status
)) {
995 *NamespaceId
= NextNamespaceId
;
999 if (NamespaceData
!= NULL
) {
1000 FreePool(NamespaceData
);
1007 Used to translate a device path node to a namespace ID.
1009 The EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL.GetNamespace() function determines the namespace ID associated with the
1010 namespace described by DevicePath.
1012 If DevicePath is a device path node type that the NVM Express Pass Thru driver supports, then the NVM Express
1013 Pass Thru driver will attempt to translate the contents DevicePath into a namespace ID.
1015 If this translation is successful, then that namespace ID is returned in NamespaceId, and EFI_SUCCESS is returned
1017 @param[in] This A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
1018 @param[in] DevicePath A pointer to the device path node that describes an NVM Express namespace on
1019 the NVM Express controller.
1020 @param[out] NamespaceId The NVM Express namespace ID contained in the device path node.
1022 @retval EFI_SUCCESS DevicePath was successfully translated to NamespaceId.
1023 @retval EFI_INVALID_PARAMETER If DevicePath or NamespaceId are NULL, then EFI_INVALID_PARAMETER is returned.
1024 @retval EFI_UNSUPPORTED If DevicePath is not a device path node type that the NVM Express Pass Thru driver
1025 supports, then EFI_UNSUPPORTED is returned.
1026 @retval EFI_NOT_FOUND If DevicePath is a device path node type that the NVM Express Pass Thru driver
1027 supports, but there is not a valid translation from DevicePath to a namespace ID,
1028 then EFI_NOT_FOUND is returned.
1032 NvmExpressGetNamespace (
1033 IN EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL
*This
,
1034 IN EFI_DEVICE_PATH_PROTOCOL
*DevicePath
,
1035 OUT UINT32
*NamespaceId
1038 NVME_NAMESPACE_DEVICE_PATH
*Node
;
1039 NVME_CONTROLLER_PRIVATE_DATA
*Private
;
1041 if ((This
== NULL
) || (DevicePath
== NULL
) || (NamespaceId
== NULL
)) {
1042 return EFI_INVALID_PARAMETER
;
1045 if (DevicePath
->Type
!= MESSAGING_DEVICE_PATH
) {
1046 return EFI_UNSUPPORTED
;
1049 Node
= (NVME_NAMESPACE_DEVICE_PATH
*)DevicePath
;
1050 Private
= NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This
);
1052 if (DevicePath
->SubType
== MSG_NVME_NAMESPACE_DP
) {
1053 if (DevicePathNodeLength(DevicePath
) != sizeof(NVME_NAMESPACE_DEVICE_PATH
)) {
1054 return EFI_NOT_FOUND
;
1058 // Check NamespaceId in the device path node is valid or not.
1060 if ((Node
->NamespaceId
== 0) ||
1061 (Node
->NamespaceId
> Private
->ControllerData
->Nn
)) {
1062 return EFI_NOT_FOUND
;
1065 *NamespaceId
= Node
->NamespaceId
;
1069 return EFI_UNSUPPORTED
;
1074 Used to allocate and build a device path node for an NVM Express namespace on an NVM Express controller.
1076 The EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL.BuildDevicePath() function allocates and builds a single device
1077 path node for the NVM Express namespace specified by NamespaceId.
1079 If the NamespaceId is not valid, then EFI_NOT_FOUND is returned.
1081 If DevicePath is NULL, then EFI_INVALID_PARAMETER is returned.
1083 If there are not enough resources to allocate the device path node, then EFI_OUT_OF_RESOURCES is returned.
1085 Otherwise, DevicePath is allocated with the boot service AllocatePool(), the contents of DevicePath are
1086 initialized to describe the NVM Express namespace specified by NamespaceId, and EFI_SUCCESS is returned.
1088 @param[in] This A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
1089 @param[in] NamespaceId The NVM Express namespace ID for which a device path node is to be
1090 allocated and built. Caller must set the NamespaceId to zero if the
1091 device path node will contain a valid UUID.
1092 @param[in,out] DevicePath A pointer to a single device path node that describes the NVM Express
1093 namespace specified by NamespaceId. This function is responsible for
1094 allocating the buffer DevicePath with the boot service AllocatePool().
1095 It is the caller's responsibility to free DevicePath when the caller
1096 is finished with DevicePath.
1097 @retval EFI_SUCCESS The device path node that describes the NVM Express namespace specified
1098 by NamespaceId was allocated and returned in DevicePath.
1099 @retval EFI_NOT_FOUND The NamespaceId is not valid.
1100 @retval EFI_INVALID_PARAMETER DevicePath is NULL.
1101 @retval EFI_OUT_OF_RESOURCES There are not enough resources to allocate the DevicePath node.
1106 NvmExpressBuildDevicePath (
1107 IN EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL
*This
,
1108 IN UINT32 NamespaceId
,
1109 IN OUT EFI_DEVICE_PATH_PROTOCOL
**DevicePath
1112 NVME_NAMESPACE_DEVICE_PATH
*Node
;
1113 NVME_CONTROLLER_PRIVATE_DATA
*Private
;
1115 NVME_ADMIN_NAMESPACE_DATA
*NamespaceData
;
1118 // Validate parameters
1120 if ((This
== NULL
) || (DevicePath
== NULL
)) {
1121 return EFI_INVALID_PARAMETER
;
1124 Status
= EFI_SUCCESS
;
1125 Private
= NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This
);
1128 // Check NamespaceId is valid or not.
1130 if ((NamespaceId
== 0) ||
1131 (NamespaceId
> Private
->ControllerData
->Nn
)) {
1132 return EFI_NOT_FOUND
;
1135 Node
= (NVME_NAMESPACE_DEVICE_PATH
*)AllocateZeroPool (sizeof (NVME_NAMESPACE_DEVICE_PATH
));
1137 return EFI_OUT_OF_RESOURCES
;
1140 Node
->Header
.Type
= MESSAGING_DEVICE_PATH
;
1141 Node
->Header
.SubType
= MSG_NVME_NAMESPACE_DP
;
1142 SetDevicePathNodeLength (&Node
->Header
, sizeof (NVME_NAMESPACE_DEVICE_PATH
));
1143 Node
->NamespaceId
= NamespaceId
;
1146 // Allocate a buffer for Identify Namespace data.
1148 NamespaceData
= NULL
;
1149 NamespaceData
= AllocateZeroPool(sizeof (NVME_ADMIN_NAMESPACE_DATA
));
1150 if(NamespaceData
== NULL
) {
1151 Status
= EFI_OUT_OF_RESOURCES
;
1156 // Get UUID from specified Identify Namespace data.
1158 Status
= NvmeIdentifyNamespace (
1161 (VOID
*)NamespaceData
1164 if (EFI_ERROR(Status
)) {
1168 Node
->NamespaceUuid
= NamespaceData
->Eui64
;
1170 *DevicePath
= (EFI_DEVICE_PATH_PROTOCOL
*)Node
;
1173 if(NamespaceData
!= NULL
) {
1174 FreePool (NamespaceData
);
1177 if (EFI_ERROR (Status
)) {