2 NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows
3 NVM Express specification.
5 (C) Copyright 2014 Hewlett-Packard Development Company, L.P.<BR>
6 Copyright (c) 2013 - 2018, Intel Corporation. All rights reserved.<BR>
7 This program and the accompanying materials
8 are licensed and made available under the terms and conditions of the BSD License
9 which accompanies this distribution. The full text of the license may be found at
10 http://opensource.org/licenses/bsd-license.php.
12 THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
13 WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
17 #include "NvmExpress.h"
20 Dump the execution status from a given completion queue entry.
22 @param[in] Cq A pointer to the NVME_CQ item.
30 DEBUG ((EFI_D_VERBOSE
, "Dump NVMe Completion Entry Status from [0x%x]:\n", Cq
));
32 DEBUG ((EFI_D_VERBOSE
, " SQ Identifier : [0x%x], Phase Tag : [%d], Cmd Identifier : [0x%x]\n", Cq
->Sqid
, Cq
->Pt
, Cq
->Cid
));
34 DEBUG ((EFI_D_VERBOSE
, " NVMe Cmd Execution Result - "));
40 DEBUG ((EFI_D_VERBOSE
, "Successful Completion\n"));
43 DEBUG ((EFI_D_VERBOSE
, "Invalid Command Opcode\n"));
46 DEBUG ((EFI_D_VERBOSE
, "Invalid Field in Command\n"));
49 DEBUG ((EFI_D_VERBOSE
, "Command ID Conflict\n"));
52 DEBUG ((EFI_D_VERBOSE
, "Data Transfer Error\n"));
55 DEBUG ((EFI_D_VERBOSE
, "Commands Aborted due to Power Loss Notification\n"));
58 DEBUG ((EFI_D_VERBOSE
, "Internal Device Error\n"));
61 DEBUG ((EFI_D_VERBOSE
, "Command Abort Requested\n"));
64 DEBUG ((EFI_D_VERBOSE
, "Command Aborted due to SQ Deletion\n"));
67 DEBUG ((EFI_D_VERBOSE
, "Command Aborted due to Failed Fused Command\n"));
70 DEBUG ((EFI_D_VERBOSE
, "Command Aborted due to Missing Fused Command\n"));
73 DEBUG ((EFI_D_VERBOSE
, "Invalid Namespace or Format\n"));
76 DEBUG ((EFI_D_VERBOSE
, "Command Sequence Error\n"));
79 DEBUG ((EFI_D_VERBOSE
, "Invalid SGL Last Segment Descriptor\n"));
82 DEBUG ((EFI_D_VERBOSE
, "Invalid Number of SGL Descriptors\n"));
85 DEBUG ((EFI_D_VERBOSE
, "Data SGL Length Invalid\n"));
88 DEBUG ((EFI_D_VERBOSE
, "Metadata SGL Length Invalid\n"));
91 DEBUG ((EFI_D_VERBOSE
, "SGL Descriptor Type Invalid\n"));
94 DEBUG ((EFI_D_VERBOSE
, "LBA Out of Range\n"));
97 DEBUG ((EFI_D_VERBOSE
, "Capacity Exceeded\n"));
100 DEBUG ((EFI_D_VERBOSE
, "Namespace Not Ready\n"));
103 DEBUG ((EFI_D_VERBOSE
, "Reservation Conflict\n"));
111 DEBUG ((EFI_D_VERBOSE
, "Completion Queue Invalid\n"));
114 DEBUG ((EFI_D_VERBOSE
, "Invalid Queue Identifier\n"));
117 DEBUG ((EFI_D_VERBOSE
, "Maximum Queue Size Exceeded\n"));
120 DEBUG ((EFI_D_VERBOSE
, "Abort Command Limit Exceeded\n"));
123 DEBUG ((EFI_D_VERBOSE
, "Asynchronous Event Request Limit Exceeded\n"));
126 DEBUG ((EFI_D_VERBOSE
, "Invalid Firmware Slot\n"));
129 DEBUG ((EFI_D_VERBOSE
, "Invalid Firmware Image\n"));
132 DEBUG ((EFI_D_VERBOSE
, "Invalid Interrupt Vector\n"));
135 DEBUG ((EFI_D_VERBOSE
, "Invalid Log Page\n"));
138 DEBUG ((EFI_D_VERBOSE
, "Invalid Format\n"));
141 DEBUG ((EFI_D_VERBOSE
, "Firmware Application Requires Conventional Reset\n"));
144 DEBUG ((EFI_D_VERBOSE
, "Invalid Queue Deletion\n"));
147 DEBUG ((EFI_D_VERBOSE
, "Feature Identifier Not Saveable\n"));
150 DEBUG ((EFI_D_VERBOSE
, "Feature Not Changeable\n"));
153 DEBUG ((EFI_D_VERBOSE
, "Feature Not Namespace Specific\n"));
156 DEBUG ((EFI_D_VERBOSE
, "Firmware Application Requires NVM Subsystem Reset\n"));
159 DEBUG ((EFI_D_VERBOSE
, "Conflicting Attributes\n"));
162 DEBUG ((EFI_D_VERBOSE
, "Invalid Protection Information\n"));
165 DEBUG ((EFI_D_VERBOSE
, "Attempted Write to Read Only Range\n"));
173 DEBUG ((EFI_D_VERBOSE
, "Write Fault\n"));
176 DEBUG ((EFI_D_VERBOSE
, "Unrecovered Read Error\n"));
179 DEBUG ((EFI_D_VERBOSE
, "End-to-end Guard Check Error\n"));
182 DEBUG ((EFI_D_VERBOSE
, "End-to-end Application Tag Check Error\n"));
185 DEBUG ((EFI_D_VERBOSE
, "End-to-end Reference Tag Check Error\n"));
188 DEBUG ((EFI_D_VERBOSE
, "Compare Failure\n"));
191 DEBUG ((EFI_D_VERBOSE
, "Access Denied\n"));
202 Create PRP lists for data transfer which is larger than 2 memory pages.
203 Note here we calcuate the number of required PRP lists and allocate them at one time.
205 @param[in] PciIo A pointer to the EFI_PCI_IO_PROTOCOL instance.
206 @param[in] PhysicalAddr The physical base address of data buffer.
207 @param[in] Pages The number of pages to be transfered.
208 @param[out] PrpListHost The host base address of PRP lists.
209 @param[in,out] PrpListNo The number of PRP List.
210 @param[out] Mapping The mapping value returned from PciIo.Map().
212 @retval The pointer to the first PRP List of the PRP lists.
217 IN EFI_PCI_IO_PROTOCOL
*PciIo
,
218 IN EFI_PHYSICAL_ADDRESS PhysicalAddr
,
220 OUT VOID
**PrpListHost
,
221 IN OUT UINTN
*PrpListNo
,
230 EFI_PHYSICAL_ADDRESS PrpListPhyAddr
;
235 // The number of Prp Entry in a memory page.
237 PrpEntryNo
= EFI_PAGE_SIZE
/ sizeof (UINT64
);
240 // Calculate total PrpList number.
242 *PrpListNo
= (UINTN
)DivU64x64Remainder ((UINT64
)Pages
, (UINT64
)PrpEntryNo
- 1, &Remainder
);
243 if (*PrpListNo
== 0) {
245 } else if ((Remainder
!= 0) && (Remainder
!= 1)) {
247 } else if (Remainder
== 1) {
248 Remainder
= PrpEntryNo
;
249 } else if (Remainder
== 0) {
250 Remainder
= PrpEntryNo
- 1;
253 Status
= PciIo
->AllocateBuffer (
262 if (EFI_ERROR (Status
)) {
266 Bytes
= EFI_PAGES_TO_SIZE (*PrpListNo
);
267 Status
= PciIo
->Map (
269 EfiPciIoOperationBusMasterCommonBuffer
,
276 if (EFI_ERROR (Status
) || (Bytes
!= EFI_PAGES_TO_SIZE (*PrpListNo
))) {
277 DEBUG ((EFI_D_ERROR
, "NvmeCreatePrpList: create PrpList failure!\n"));
281 // Fill all PRP lists except of last one.
283 ZeroMem (*PrpListHost
, Bytes
);
284 for (PrpListIndex
= 0; PrpListIndex
< *PrpListNo
- 1; ++PrpListIndex
) {
285 PrpListBase
= *(UINT64
*)PrpListHost
+ PrpListIndex
* EFI_PAGE_SIZE
;
287 for (PrpEntryIndex
= 0; PrpEntryIndex
< PrpEntryNo
; ++PrpEntryIndex
) {
288 if (PrpEntryIndex
!= PrpEntryNo
- 1) {
290 // Fill all PRP entries except of last one.
292 *((UINT64
*)(UINTN
)PrpListBase
+ PrpEntryIndex
) = PhysicalAddr
;
293 PhysicalAddr
+= EFI_PAGE_SIZE
;
296 // Fill last PRP entries with next PRP List pointer.
298 *((UINT64
*)(UINTN
)PrpListBase
+ PrpEntryIndex
) = PrpListPhyAddr
+ (PrpListIndex
+ 1) * EFI_PAGE_SIZE
;
303 // Fill last PRP list.
305 PrpListBase
= *(UINT64
*)PrpListHost
+ PrpListIndex
* EFI_PAGE_SIZE
;
306 for (PrpEntryIndex
= 0; PrpEntryIndex
< Remainder
; ++PrpEntryIndex
) {
307 *((UINT64
*)(UINTN
)PrpListBase
+ PrpEntryIndex
) = PhysicalAddr
;
308 PhysicalAddr
+= EFI_PAGE_SIZE
;
311 return (VOID
*)(UINTN
)PrpListPhyAddr
;
314 PciIo
->FreeBuffer (PciIo
, *PrpListNo
, *PrpListHost
);
320 Aborts the asynchronous PassThru requests.
322 @param[in] Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA
325 @retval EFI_SUCCESS The asynchronous PassThru requests have been aborted.
326 @return EFI_DEVICE_ERROR Fail to abort all the asynchronous PassThru requests.
330 AbortAsyncPassThruTasks (
331 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
334 EFI_PCI_IO_PROTOCOL
*PciIo
;
336 LIST_ENTRY
*NextLink
;
337 NVME_BLKIO2_SUBTASK
*Subtask
;
338 NVME_BLKIO2_REQUEST
*BlkIo2Request
;
339 NVME_PASS_THRU_ASYNC_REQ
*AsyncRequest
;
340 EFI_BLOCK_IO2_TOKEN
*Token
;
344 PciIo
= Private
->PciIo
;
345 OldTpl
= gBS
->RaiseTPL (TPL_NOTIFY
);
348 // Cancel the unsubmitted subtasks.
350 for (Link
= GetFirstNode (&Private
->UnsubmittedSubtasks
);
351 !IsNull (&Private
->UnsubmittedSubtasks
, Link
);
353 NextLink
= GetNextNode (&Private
->UnsubmittedSubtasks
, Link
);
354 Subtask
= NVME_BLKIO2_SUBTASK_FROM_LINK (Link
);
355 BlkIo2Request
= Subtask
->BlockIo2Request
;
356 Token
= BlkIo2Request
->Token
;
358 BlkIo2Request
->UnsubmittedSubtaskNum
--;
359 if (Subtask
->IsLast
) {
360 BlkIo2Request
->LastSubtaskSubmitted
= TRUE
;
362 Token
->TransactionStatus
= EFI_ABORTED
;
364 RemoveEntryList (Link
);
365 InsertTailList (&BlkIo2Request
->SubtasksQueue
, Link
);
366 gBS
->SignalEvent (Subtask
->Event
);
370 // Cleanup the resources for the asynchronous PassThru requests.
372 for (Link
= GetFirstNode (&Private
->AsyncPassThruQueue
);
373 !IsNull (&Private
->AsyncPassThruQueue
, Link
);
375 NextLink
= GetNextNode (&Private
->AsyncPassThruQueue
, Link
);
376 AsyncRequest
= NVME_PASS_THRU_ASYNC_REQ_FROM_THIS (Link
);
378 if (AsyncRequest
->MapData
!= NULL
) {
379 PciIo
->Unmap (PciIo
, AsyncRequest
->MapData
);
381 if (AsyncRequest
->MapMeta
!= NULL
) {
382 PciIo
->Unmap (PciIo
, AsyncRequest
->MapMeta
);
384 if (AsyncRequest
->MapPrpList
!= NULL
) {
385 PciIo
->Unmap (PciIo
, AsyncRequest
->MapPrpList
);
387 if (AsyncRequest
->PrpListHost
!= NULL
) {
390 AsyncRequest
->PrpListNo
,
391 AsyncRequest
->PrpListHost
395 RemoveEntryList (Link
);
396 gBS
->SignalEvent (AsyncRequest
->CallerEvent
);
397 FreePool (AsyncRequest
);
400 if (IsListEmpty (&Private
->AsyncPassThruQueue
) &&
401 IsListEmpty (&Private
->UnsubmittedSubtasks
)) {
402 Status
= EFI_SUCCESS
;
404 Status
= EFI_DEVICE_ERROR
;
407 gBS
->RestoreTPL (OldTpl
);
414 Sends an NVM Express Command Packet to an NVM Express controller or namespace. This function supports
415 both blocking I/O and non-blocking I/O. The blocking I/O functionality is required, and the non-blocking
416 I/O functionality is optional.
419 @param[in] This A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
420 @param[in] NamespaceId A 32 bit namespace ID as defined in the NVMe specification to which the NVM Express Command
421 Packet will be sent. A value of 0 denotes the NVM Express controller, a value of all 0xFF's
422 (all bytes are 0xFF) in the namespace ID specifies that the command packet should be sent to
423 all valid namespaces.
424 @param[in,out] Packet A pointer to the NVM Express Command Packet.
425 @param[in] Event If non-blocking I/O is not supported then Event is ignored, and blocking I/O is performed.
426 If Event is NULL, then blocking I/O is performed. If Event is not NULL and non-blocking I/O
427 is supported, then non-blocking I/O is performed, and Event will be signaled when the NVM
428 Express Command Packet completes.
430 @retval EFI_SUCCESS The NVM Express Command Packet was sent by the host. TransferLength bytes were transferred
431 to, or from DataBuffer.
432 @retval EFI_BAD_BUFFER_SIZE The NVM Express Command Packet was not executed. The number of bytes that could be transferred
433 is returned in TransferLength.
434 @retval EFI_NOT_READY The NVM Express Command Packet could not be sent because the controller is not ready. The caller
435 may retry again later.
436 @retval EFI_DEVICE_ERROR A device error occurred while attempting to send the NVM Express Command Packet.
437 @retval EFI_INVALID_PARAMETER NamespaceId or the contents of EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET are invalid. The NVM
438 Express Command Packet was not sent, so no additional status information is available.
439 @retval EFI_UNSUPPORTED The command described by the NVM Express Command Packet is not supported by the NVM Express
440 controller. The NVM Express Command Packet was not sent so no additional status information
442 @retval EFI_TIMEOUT A timeout occurred while waiting for the NVM Express Command Packet to execute.
448 IN EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL
*This
,
449 IN UINT32 NamespaceId
,
450 IN OUT EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET
*Packet
,
451 IN EFI_EVENT Event OPTIONAL
454 NVME_CONTROLLER_PRIVATE_DATA
*Private
;
456 EFI_STATUS PreviousStatus
;
457 EFI_PCI_IO_PROTOCOL
*PciIo
;
463 EFI_EVENT TimerEvent
;
464 EFI_PCI_IO_PROTOCOL_OPERATION Flag
;
465 EFI_PHYSICAL_ADDRESS PhyAddr
;
477 NVME_PASS_THRU_ASYNC_REQ
*AsyncRequest
;
481 // check the data fields in Packet parameter.
483 if ((This
== NULL
) || (Packet
== NULL
)) {
484 return EFI_INVALID_PARAMETER
;
487 if ((Packet
->NvmeCmd
== NULL
) || (Packet
->NvmeCompletion
== NULL
)) {
488 return EFI_INVALID_PARAMETER
;
491 if (Packet
->QueueType
!= NVME_ADMIN_QUEUE
&& Packet
->QueueType
!= NVME_IO_QUEUE
) {
492 return EFI_INVALID_PARAMETER
;
496 // 'Attributes' with neither EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL nor
497 // EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL set is an illegal
500 Attributes
= This
->Mode
->Attributes
;
501 if ((Attributes
& (EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL
|
502 EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL
)) == 0) {
503 return EFI_INVALID_PARAMETER
;
507 // Buffer alignment check for TransferBuffer & MetadataBuffer.
509 IoAlign
= This
->Mode
->IoAlign
;
510 if (IoAlign
> 0 && (((UINTN
) Packet
->TransferBuffer
& (IoAlign
- 1)) != 0)) {
511 return EFI_INVALID_PARAMETER
;
514 if (IoAlign
> 0 && (((UINTN
) Packet
->MetadataBuffer
& (IoAlign
- 1)) != 0)) {
515 return EFI_INVALID_PARAMETER
;
518 Private
= NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This
);
521 // Check NamespaceId is valid or not.
523 if ((NamespaceId
> Private
->ControllerData
->Nn
) &&
524 (NamespaceId
!= (UINT32
) -1)) {
525 return EFI_INVALID_PARAMETER
;
529 // Check whether TransferLength exceeds the maximum data transfer size.
531 if (Private
->ControllerData
->Mdts
!= 0) {
532 MaxTransLen
= (1 << (Private
->ControllerData
->Mdts
)) *
533 (1 << (Private
->Cap
.Mpsmin
+ 12));
534 if (Packet
->TransferLength
> MaxTransLen
) {
535 Packet
->TransferLength
= MaxTransLen
;
536 return EFI_BAD_BUFFER_SIZE
;
540 PciIo
= Private
->PciIo
;
548 Status
= EFI_SUCCESS
;
550 if (Packet
->QueueType
== NVME_ADMIN_QUEUE
) {
559 // Submission queue full check.
561 if ((Private
->SqTdbl
[QueueId
].Sqt
+ 1) % (NVME_ASYNC_CSQ_SIZE
+ 1) ==
562 Private
->AsyncSqHead
) {
563 return EFI_NOT_READY
;
567 Sq
= Private
->SqBuffer
[QueueId
] + Private
->SqTdbl
[QueueId
].Sqt
;
568 Cq
= Private
->CqBuffer
[QueueId
] + Private
->CqHdbl
[QueueId
].Cqh
;
570 if (Packet
->NvmeCmd
->Nsid
!= NamespaceId
) {
571 return EFI_INVALID_PARAMETER
;
574 ZeroMem (Sq
, sizeof (NVME_SQ
));
575 Sq
->Opc
= (UINT8
)Packet
->NvmeCmd
->Cdw0
.Opcode
;
576 Sq
->Fuse
= (UINT8
)Packet
->NvmeCmd
->Cdw0
.FusedOperation
;
577 Sq
->Cid
= Private
->Cid
[QueueId
]++;
578 Sq
->Nsid
= Packet
->NvmeCmd
->Nsid
;
581 // Currently we only support PRP for data transfer, SGL is NOT supported.
583 ASSERT (Sq
->Psdt
== 0);
585 DEBUG ((EFI_D_ERROR
, "NvmExpressPassThru: doesn't support SGL mechanism\n"));
586 return EFI_UNSUPPORTED
;
589 Sq
->Prp
[0] = (UINT64
)(UINTN
)Packet
->TransferBuffer
;
590 if ((Packet
->QueueType
== NVME_ADMIN_QUEUE
) &&
591 ((Sq
->Opc
== NVME_ADMIN_CRIOCQ_CMD
) || (Sq
->Opc
== NVME_ADMIN_CRIOSQ_CMD
))) {
593 // Currently, we only use the IO Completion/Submission queues created internally
594 // by this driver during controller initialization. Any other IO queues created
595 // will not be consumed here. The value is little to accept external IO queue
596 // creation requests, so here we will return EFI_UNSUPPORTED for external IO
597 // queue creation request.
599 if (!Private
->CreateIoQueue
) {
600 DEBUG ((DEBUG_ERROR
, "NvmExpressPassThru: Does not support external IO queues creation request.\n"));
601 return EFI_UNSUPPORTED
;
603 } else if ((Sq
->Opc
& (BIT0
| BIT1
)) != 0) {
605 // If the NVMe cmd has data in or out, then mapping the user buffer to the PCI controller specific addresses.
607 if (((Packet
->TransferLength
!= 0) && (Packet
->TransferBuffer
== NULL
)) ||
608 ((Packet
->TransferLength
== 0) && (Packet
->TransferBuffer
!= NULL
))) {
609 return EFI_INVALID_PARAMETER
;
612 if ((Sq
->Opc
& BIT0
) != 0) {
613 Flag
= EfiPciIoOperationBusMasterRead
;
615 Flag
= EfiPciIoOperationBusMasterWrite
;
618 if ((Packet
->TransferLength
!= 0) && (Packet
->TransferBuffer
!= NULL
)) {
619 MapLength
= Packet
->TransferLength
;
620 Status
= PciIo
->Map (
623 Packet
->TransferBuffer
,
628 if (EFI_ERROR (Status
) || (Packet
->TransferLength
!= MapLength
)) {
629 return EFI_OUT_OF_RESOURCES
;
632 Sq
->Prp
[0] = PhyAddr
;
636 if((Packet
->MetadataLength
!= 0) && (Packet
->MetadataBuffer
!= NULL
)) {
637 MapLength
= Packet
->MetadataLength
;
638 Status
= PciIo
->Map (
641 Packet
->MetadataBuffer
,
646 if (EFI_ERROR (Status
) || (Packet
->MetadataLength
!= MapLength
)) {
652 return EFI_OUT_OF_RESOURCES
;
658 // If the buffer size spans more than two memory pages (page size as defined in CC.Mps),
659 // then build a PRP list in the second PRP submission queue entry.
661 Offset
= ((UINT16
)Sq
->Prp
[0]) & (EFI_PAGE_SIZE
- 1);
662 Bytes
= Packet
->TransferLength
;
664 if ((Offset
+ Bytes
) > (EFI_PAGE_SIZE
* 2)) {
666 // Create PrpList for remaining data buffer.
668 PhyAddr
= (Sq
->Prp
[0] + EFI_PAGE_SIZE
) & ~(EFI_PAGE_SIZE
- 1);
669 Prp
= NvmeCreatePrpList (PciIo
, PhyAddr
, EFI_SIZE_TO_PAGES(Offset
+ Bytes
) - 1, &PrpListHost
, &PrpListNo
, &MapPrpList
);
671 Status
= EFI_OUT_OF_RESOURCES
;
675 Sq
->Prp
[1] = (UINT64
)(UINTN
)Prp
;
676 } else if ((Offset
+ Bytes
) > EFI_PAGE_SIZE
) {
677 Sq
->Prp
[1] = (Sq
->Prp
[0] + EFI_PAGE_SIZE
) & ~(EFI_PAGE_SIZE
- 1);
680 if(Packet
->NvmeCmd
->Flags
& CDW2_VALID
) {
681 Sq
->Rsvd2
= (UINT64
)Packet
->NvmeCmd
->Cdw2
;
683 if(Packet
->NvmeCmd
->Flags
& CDW3_VALID
) {
684 Sq
->Rsvd2
|= LShiftU64 ((UINT64
)Packet
->NvmeCmd
->Cdw3
, 32);
686 if(Packet
->NvmeCmd
->Flags
& CDW10_VALID
) {
687 Sq
->Payload
.Raw
.Cdw10
= Packet
->NvmeCmd
->Cdw10
;
689 if(Packet
->NvmeCmd
->Flags
& CDW11_VALID
) {
690 Sq
->Payload
.Raw
.Cdw11
= Packet
->NvmeCmd
->Cdw11
;
692 if(Packet
->NvmeCmd
->Flags
& CDW12_VALID
) {
693 Sq
->Payload
.Raw
.Cdw12
= Packet
->NvmeCmd
->Cdw12
;
695 if(Packet
->NvmeCmd
->Flags
& CDW13_VALID
) {
696 Sq
->Payload
.Raw
.Cdw13
= Packet
->NvmeCmd
->Cdw13
;
698 if(Packet
->NvmeCmd
->Flags
& CDW14_VALID
) {
699 Sq
->Payload
.Raw
.Cdw14
= Packet
->NvmeCmd
->Cdw14
;
701 if(Packet
->NvmeCmd
->Flags
& CDW15_VALID
) {
702 Sq
->Payload
.Raw
.Cdw15
= Packet
->NvmeCmd
->Cdw15
;
706 // Ring the submission queue doorbell.
708 if ((Event
!= NULL
) && (QueueId
!= 0)) {
709 Private
->SqTdbl
[QueueId
].Sqt
=
710 (Private
->SqTdbl
[QueueId
].Sqt
+ 1) % (NVME_ASYNC_CSQ_SIZE
+ 1);
712 Private
->SqTdbl
[QueueId
].Sqt
^= 1;
714 Data
= ReadUnaligned32 ((UINT32
*)&Private
->SqTdbl
[QueueId
]);
715 Status
= PciIo
->Mem
.Write (
719 NVME_SQTDBL_OFFSET(QueueId
, Private
->Cap
.Dstrd
),
724 if (EFI_ERROR (Status
)) {
729 // For non-blocking requests, return directly if the command is placed
730 // in the submission queue.
732 if ((Event
!= NULL
) && (QueueId
!= 0)) {
733 AsyncRequest
= AllocateZeroPool (sizeof (NVME_PASS_THRU_ASYNC_REQ
));
734 if (AsyncRequest
== NULL
) {
735 Status
= EFI_DEVICE_ERROR
;
739 AsyncRequest
->Signature
= NVME_PASS_THRU_ASYNC_REQ_SIG
;
740 AsyncRequest
->Packet
= Packet
;
741 AsyncRequest
->CommandId
= Sq
->Cid
;
742 AsyncRequest
->CallerEvent
= Event
;
743 AsyncRequest
->MapData
= MapData
;
744 AsyncRequest
->MapMeta
= MapMeta
;
745 AsyncRequest
->MapPrpList
= MapPrpList
;
746 AsyncRequest
->PrpListNo
= PrpListNo
;
747 AsyncRequest
->PrpListHost
= PrpListHost
;
749 OldTpl
= gBS
->RaiseTPL (TPL_NOTIFY
);
750 InsertTailList (&Private
->AsyncPassThruQueue
, &AsyncRequest
->Link
);
751 gBS
->RestoreTPL (OldTpl
);
756 Status
= gBS
->CreateEvent (
763 if (EFI_ERROR (Status
)) {
767 Status
= gBS
->SetTimer(TimerEvent
, TimerRelative
, Packet
->CommandTimeout
);
769 if (EFI_ERROR(Status
)) {
774 // Wait for completion queue to get filled in.
776 Status
= EFI_TIMEOUT
;
777 while (EFI_ERROR (gBS
->CheckEvent (TimerEvent
))) {
778 if (Cq
->Pt
!= Private
->Pt
[QueueId
]) {
779 Status
= EFI_SUCCESS
;
785 // Check the NVMe cmd execution result
787 if (Status
!= EFI_TIMEOUT
) {
788 if ((Cq
->Sct
== 0) && (Cq
->Sc
== 0)) {
789 Status
= EFI_SUCCESS
;
791 Status
= EFI_DEVICE_ERROR
;
793 // Dump every completion entry status for debugging.
800 // Copy the Respose Queue entry for this command to the callers response buffer
802 CopyMem(Packet
->NvmeCompletion
, Cq
, sizeof(EFI_NVM_EXPRESS_COMPLETION
));
805 // Timeout occurs for an NVMe command. Reset the controller to abort the
806 // outstanding commands.
808 DEBUG ((DEBUG_ERROR
, "NvmExpressPassThru: Timeout occurs for an NVMe command.\n"));
811 // Disable the timer to trigger the process of async transfers temporarily.
813 Status
= gBS
->SetTimer (Private
->TimerEvent
, TimerCancel
, 0);
814 if (EFI_ERROR (Status
)) {
819 // Reset the NVMe controller.
821 Status
= NvmeControllerInit (Private
);
822 if (!EFI_ERROR (Status
)) {
823 Status
= AbortAsyncPassThruTasks (Private
);
824 if (!EFI_ERROR (Status
)) {
826 // Re-enable the timer to trigger the process of async transfers.
828 Status
= gBS
->SetTimer (Private
->TimerEvent
, TimerPeriodic
, NVME_HC_ASYNC_TIMER
);
829 if (!EFI_ERROR (Status
)) {
831 // Return EFI_TIMEOUT to indicate a timeout occurs for NVMe PassThru command.
833 Status
= EFI_TIMEOUT
;
837 Status
= EFI_DEVICE_ERROR
;
843 if ((Private
->CqHdbl
[QueueId
].Cqh
^= 1) == 0) {
844 Private
->Pt
[QueueId
] ^= 1;
847 Data
= ReadUnaligned32 ((UINT32
*)&Private
->CqHdbl
[QueueId
]);
848 PreviousStatus
= Status
;
849 Status
= PciIo
->Mem
.Write (
853 NVME_CQHDBL_OFFSET(QueueId
, Private
->Cap
.Dstrd
),
857 // The return status of PciIo->Mem.Write should not override
858 // previous status if previous status contains error.
859 Status
= EFI_ERROR (PreviousStatus
) ? PreviousStatus
: Status
;
862 // For now, the code does not support the non-blocking feature for admin queue.
863 // If Event is not NULL for admin queue, signal the caller's event here.
866 ASSERT (QueueId
== 0);
867 gBS
->SignalEvent (Event
);
871 if (MapData
!= NULL
) {
878 if (MapMeta
!= NULL
) {
885 if (MapPrpList
!= NULL
) {
893 PciIo
->FreeBuffer (PciIo
, PrpListNo
, PrpListHost
);
896 if (TimerEvent
!= NULL
) {
897 gBS
->CloseEvent (TimerEvent
);
903 Used to retrieve the next namespace ID for this NVM Express controller.
905 The EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL.GetNextNamespace() function retrieves the next valid
906 namespace ID on this NVM Express controller.
908 If on input the value pointed to by NamespaceId is 0xFFFFFFFF, then the first valid namespace
909 ID defined on the NVM Express controller is returned in the location pointed to by NamespaceId
910 and a status of EFI_SUCCESS is returned.
912 If on input the value pointed to by NamespaceId is an invalid namespace ID other than 0xFFFFFFFF,
913 then EFI_INVALID_PARAMETER is returned.
915 If on input the value pointed to by NamespaceId is a valid namespace ID, then the next valid
916 namespace ID on the NVM Express controller is returned in the location pointed to by NamespaceId,
917 and EFI_SUCCESS is returned.
919 If the value pointed to by NamespaceId is the namespace ID of the last namespace on the NVM
920 Express controller, then EFI_NOT_FOUND is returned.
922 @param[in] This A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
923 @param[in,out] NamespaceId On input, a pointer to a legal NamespaceId for an NVM Express
924 namespace present on the NVM Express controller. On output, a
925 pointer to the next NamespaceId of an NVM Express namespace on
926 an NVM Express controller. An input value of 0xFFFFFFFF retrieves
927 the first NamespaceId for an NVM Express namespace present on an
928 NVM Express controller.
930 @retval EFI_SUCCESS The Namespace ID of the next Namespace was returned.
931 @retval EFI_NOT_FOUND There are no more namespaces defined on this controller.
932 @retval EFI_INVALID_PARAMETER NamespaceId is an invalid value other than 0xFFFFFFFF.
937 NvmExpressGetNextNamespace (
938 IN EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL
*This
,
939 IN OUT UINT32
*NamespaceId
942 NVME_CONTROLLER_PRIVATE_DATA
*Private
;
943 NVME_ADMIN_NAMESPACE_DATA
*NamespaceData
;
944 UINT32 NextNamespaceId
;
947 if ((This
== NULL
) || (NamespaceId
== NULL
)) {
948 return EFI_INVALID_PARAMETER
;
951 NamespaceData
= NULL
;
952 Status
= EFI_NOT_FOUND
;
954 Private
= NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This
);
956 // If the NamespaceId input value is 0xFFFFFFFF, then get the first valid namespace ID
958 if (*NamespaceId
== 0xFFFFFFFF) {
960 // Start with the first namespace ID
964 // Allocate buffer for Identify Namespace data.
966 NamespaceData
= (NVME_ADMIN_NAMESPACE_DATA
*)AllocateZeroPool (sizeof (NVME_ADMIN_NAMESPACE_DATA
));
968 if (NamespaceData
== NULL
) {
969 return EFI_NOT_FOUND
;
972 Status
= NvmeIdentifyNamespace (Private
, NextNamespaceId
, NamespaceData
);
973 if (EFI_ERROR(Status
)) {
977 *NamespaceId
= NextNamespaceId
;
979 if (*NamespaceId
> Private
->ControllerData
->Nn
) {
980 return EFI_INVALID_PARAMETER
;
983 NextNamespaceId
= *NamespaceId
+ 1;
984 if (NextNamespaceId
> Private
->ControllerData
->Nn
) {
985 return EFI_NOT_FOUND
;
989 // Allocate buffer for Identify Namespace data.
991 NamespaceData
= (NVME_ADMIN_NAMESPACE_DATA
*)AllocateZeroPool (sizeof (NVME_ADMIN_NAMESPACE_DATA
));
992 if (NamespaceData
== NULL
) {
993 return EFI_NOT_FOUND
;
996 Status
= NvmeIdentifyNamespace (Private
, NextNamespaceId
, NamespaceData
);
997 if (EFI_ERROR(Status
)) {
1001 *NamespaceId
= NextNamespaceId
;
1005 if (NamespaceData
!= NULL
) {
1006 FreePool(NamespaceData
);
1013 Used to translate a device path node to a namespace ID.
1015 The EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL.GetNamespace() function determines the namespace ID associated with the
1016 namespace described by DevicePath.
1018 If DevicePath is a device path node type that the NVM Express Pass Thru driver supports, then the NVM Express
1019 Pass Thru driver will attempt to translate the contents DevicePath into a namespace ID.
1021 If this translation is successful, then that namespace ID is returned in NamespaceId, and EFI_SUCCESS is returned
1023 @param[in] This A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
1024 @param[in] DevicePath A pointer to the device path node that describes an NVM Express namespace on
1025 the NVM Express controller.
1026 @param[out] NamespaceId The NVM Express namespace ID contained in the device path node.
1028 @retval EFI_SUCCESS DevicePath was successfully translated to NamespaceId.
1029 @retval EFI_INVALID_PARAMETER If DevicePath or NamespaceId are NULL, then EFI_INVALID_PARAMETER is returned.
1030 @retval EFI_UNSUPPORTED If DevicePath is not a device path node type that the NVM Express Pass Thru driver
1031 supports, then EFI_UNSUPPORTED is returned.
1032 @retval EFI_NOT_FOUND If DevicePath is a device path node type that the NVM Express Pass Thru driver
1033 supports, but there is not a valid translation from DevicePath to a namespace ID,
1034 then EFI_NOT_FOUND is returned.
1038 NvmExpressGetNamespace (
1039 IN EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL
*This
,
1040 IN EFI_DEVICE_PATH_PROTOCOL
*DevicePath
,
1041 OUT UINT32
*NamespaceId
1044 NVME_NAMESPACE_DEVICE_PATH
*Node
;
1045 NVME_CONTROLLER_PRIVATE_DATA
*Private
;
1047 if ((This
== NULL
) || (DevicePath
== NULL
) || (NamespaceId
== NULL
)) {
1048 return EFI_INVALID_PARAMETER
;
1051 if (DevicePath
->Type
!= MESSAGING_DEVICE_PATH
) {
1052 return EFI_UNSUPPORTED
;
1055 Node
= (NVME_NAMESPACE_DEVICE_PATH
*)DevicePath
;
1056 Private
= NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This
);
1058 if (DevicePath
->SubType
== MSG_NVME_NAMESPACE_DP
) {
1059 if (DevicePathNodeLength(DevicePath
) != sizeof(NVME_NAMESPACE_DEVICE_PATH
)) {
1060 return EFI_NOT_FOUND
;
1064 // Check NamespaceId in the device path node is valid or not.
1066 if ((Node
->NamespaceId
== 0) ||
1067 (Node
->NamespaceId
> Private
->ControllerData
->Nn
)) {
1068 return EFI_NOT_FOUND
;
1071 *NamespaceId
= Node
->NamespaceId
;
1075 return EFI_UNSUPPORTED
;
1080 Used to allocate and build a device path node for an NVM Express namespace on an NVM Express controller.
1082 The EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL.BuildDevicePath() function allocates and builds a single device
1083 path node for the NVM Express namespace specified by NamespaceId.
1085 If the NamespaceId is not valid, then EFI_NOT_FOUND is returned.
1087 If DevicePath is NULL, then EFI_INVALID_PARAMETER is returned.
1089 If there are not enough resources to allocate the device path node, then EFI_OUT_OF_RESOURCES is returned.
1091 Otherwise, DevicePath is allocated with the boot service AllocatePool(), the contents of DevicePath are
1092 initialized to describe the NVM Express namespace specified by NamespaceId, and EFI_SUCCESS is returned.
1094 @param[in] This A pointer to the EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL instance.
1095 @param[in] NamespaceId The NVM Express namespace ID for which a device path node is to be
1096 allocated and built. Caller must set the NamespaceId to zero if the
1097 device path node will contain a valid UUID.
1098 @param[in,out] DevicePath A pointer to a single device path node that describes the NVM Express
1099 namespace specified by NamespaceId. This function is responsible for
1100 allocating the buffer DevicePath with the boot service AllocatePool().
1101 It is the caller's responsibility to free DevicePath when the caller
1102 is finished with DevicePath.
1103 @retval EFI_SUCCESS The device path node that describes the NVM Express namespace specified
1104 by NamespaceId was allocated and returned in DevicePath.
1105 @retval EFI_NOT_FOUND The NamespaceId is not valid.
1106 @retval EFI_INVALID_PARAMETER DevicePath is NULL.
1107 @retval EFI_OUT_OF_RESOURCES There are not enough resources to allocate the DevicePath node.
1112 NvmExpressBuildDevicePath (
1113 IN EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL
*This
,
1114 IN UINT32 NamespaceId
,
1115 IN OUT EFI_DEVICE_PATH_PROTOCOL
**DevicePath
1118 NVME_NAMESPACE_DEVICE_PATH
*Node
;
1119 NVME_CONTROLLER_PRIVATE_DATA
*Private
;
1121 NVME_ADMIN_NAMESPACE_DATA
*NamespaceData
;
1124 // Validate parameters
1126 if ((This
== NULL
) || (DevicePath
== NULL
)) {
1127 return EFI_INVALID_PARAMETER
;
1130 Status
= EFI_SUCCESS
;
1131 Private
= NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This
);
1134 // Check NamespaceId is valid or not.
1136 if ((NamespaceId
== 0) ||
1137 (NamespaceId
> Private
->ControllerData
->Nn
)) {
1138 return EFI_NOT_FOUND
;
1141 Node
= (NVME_NAMESPACE_DEVICE_PATH
*)AllocateZeroPool (sizeof (NVME_NAMESPACE_DEVICE_PATH
));
1143 return EFI_OUT_OF_RESOURCES
;
1146 Node
->Header
.Type
= MESSAGING_DEVICE_PATH
;
1147 Node
->Header
.SubType
= MSG_NVME_NAMESPACE_DP
;
1148 SetDevicePathNodeLength (&Node
->Header
, sizeof (NVME_NAMESPACE_DEVICE_PATH
));
1149 Node
->NamespaceId
= NamespaceId
;
1152 // Allocate a buffer for Identify Namespace data.
1154 NamespaceData
= NULL
;
1155 NamespaceData
= AllocateZeroPool(sizeof (NVME_ADMIN_NAMESPACE_DATA
));
1156 if(NamespaceData
== NULL
) {
1157 Status
= EFI_OUT_OF_RESOURCES
;
1162 // Get UUID from specified Identify Namespace data.
1164 Status
= NvmeIdentifyNamespace (
1167 (VOID
*)NamespaceData
1170 if (EFI_ERROR(Status
)) {
1174 Node
->NamespaceUuid
= NamespaceData
->Eui64
;
1176 *DevicePath
= (EFI_DEVICE_PATH_PROTOCOL
*)Node
;
1179 if(NamespaceData
!= NULL
) {
1180 FreePool (NamespaceData
);
1183 if (EFI_ERROR (Status
)) {