2 NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows
3 NVM Express specification.
5 Copyright (c) 2013 - 2018, Intel Corporation. All rights reserved.<BR>
6 This program and the accompanying materials
7 are licensed and made available under the terms and conditions of the BSD License
8 which accompanies this distribution. The full text of the license may be found at
9 http://opensource.org/licenses/bsd-license.php.
11 THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
12 WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
16 #include "NvmExpress.h"
18 #define NVME_SHUTDOWN_PROCESS_TIMEOUT 45
21 // The number of NVME controllers managed by this driver, used by
22 // NvmeRegisterShutdownNotification() and NvmeUnregisterShutdownNotification().
24 UINTN mNvmeControllerNumber
= 0;
27 Read Nvm Express controller capability register.
29 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
30 @param Cap The buffer used to store capability register content.
32 @return EFI_SUCCESS Successfully read the controller capability register content.
33 @return EFI_DEVICE_ERROR Fail to read the controller capability register.
37 ReadNvmeControllerCapabilities (
38 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
,
42 EFI_PCI_IO_PROTOCOL
*PciIo
;
46 PciIo
= Private
->PciIo
;
47 Status
= PciIo
->Mem
.Read (
56 if (EFI_ERROR(Status
)) {
60 WriteUnaligned64 ((UINT64
*)Cap
, Data
);
65 Read Nvm Express controller configuration register.
67 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
68 @param Cc The buffer used to store configuration register content.
70 @return EFI_SUCCESS Successfully read the controller configuration register content.
71 @return EFI_DEVICE_ERROR Fail to read the controller configuration register.
75 ReadNvmeControllerConfiguration (
76 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
,
80 EFI_PCI_IO_PROTOCOL
*PciIo
;
84 PciIo
= Private
->PciIo
;
85 Status
= PciIo
->Mem
.Read (
94 if (EFI_ERROR(Status
)) {
98 WriteUnaligned32 ((UINT32
*)Cc
, Data
);
103 Write Nvm Express controller configuration register.
105 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
106 @param Cc The buffer used to store the content to be written into configuration register.
108 @return EFI_SUCCESS Successfully write data into the controller configuration register.
109 @return EFI_DEVICE_ERROR Fail to write data into the controller configuration register.
113 WriteNvmeControllerConfiguration (
114 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
,
118 EFI_PCI_IO_PROTOCOL
*PciIo
;
122 PciIo
= Private
->PciIo
;
123 Data
= ReadUnaligned32 ((UINT32
*)Cc
);
124 Status
= PciIo
->Mem
.Write (
133 if (EFI_ERROR(Status
)) {
137 DEBUG ((EFI_D_INFO
, "Cc.En: %d\n", Cc
->En
));
138 DEBUG ((EFI_D_INFO
, "Cc.Css: %d\n", Cc
->Css
));
139 DEBUG ((EFI_D_INFO
, "Cc.Mps: %d\n", Cc
->Mps
));
140 DEBUG ((EFI_D_INFO
, "Cc.Ams: %d\n", Cc
->Ams
));
141 DEBUG ((EFI_D_INFO
, "Cc.Shn: %d\n", Cc
->Shn
));
142 DEBUG ((EFI_D_INFO
, "Cc.Iosqes: %d\n", Cc
->Iosqes
));
143 DEBUG ((EFI_D_INFO
, "Cc.Iocqes: %d\n", Cc
->Iocqes
));
149 Read Nvm Express controller status register.
151 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
152 @param Csts The buffer used to store status register content.
154 @return EFI_SUCCESS Successfully read the controller status register content.
155 @return EFI_DEVICE_ERROR Fail to read the controller status register.
159 ReadNvmeControllerStatus (
160 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
,
164 EFI_PCI_IO_PROTOCOL
*PciIo
;
168 PciIo
= Private
->PciIo
;
169 Status
= PciIo
->Mem
.Read (
178 if (EFI_ERROR(Status
)) {
182 WriteUnaligned32 ((UINT32
*)Csts
, Data
);
187 Read Nvm Express admin queue attributes register.
189 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
190 @param Aqa The buffer used to store admin queue attributes register content.
192 @return EFI_SUCCESS Successfully read the admin queue attributes register content.
193 @return EFI_DEVICE_ERROR Fail to read the admin queue attributes register.
197 ReadNvmeAdminQueueAttributes (
198 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
,
202 EFI_PCI_IO_PROTOCOL
*PciIo
;
206 PciIo
= Private
->PciIo
;
207 Status
= PciIo
->Mem
.Read (
216 if (EFI_ERROR(Status
)) {
220 WriteUnaligned32 ((UINT32
*)Aqa
, Data
);
225 Write Nvm Express admin queue attributes register.
227 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
228 @param Aqa The buffer used to store the content to be written into admin queue attributes register.
230 @return EFI_SUCCESS Successfully write data into the admin queue attributes register.
231 @return EFI_DEVICE_ERROR Fail to write data into the admin queue attributes register.
235 WriteNvmeAdminQueueAttributes (
236 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
,
240 EFI_PCI_IO_PROTOCOL
*PciIo
;
244 PciIo
= Private
->PciIo
;
245 Data
= ReadUnaligned32 ((UINT32
*)Aqa
);
246 Status
= PciIo
->Mem
.Write (
255 if (EFI_ERROR(Status
)) {
259 DEBUG ((EFI_D_INFO
, "Aqa.Asqs: %d\n", Aqa
->Asqs
));
260 DEBUG ((EFI_D_INFO
, "Aqa.Acqs: %d\n", Aqa
->Acqs
));
266 Read Nvm Express admin submission queue base address register.
268 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
269 @param Asq The buffer used to store admin submission queue base address register content.
271 @return EFI_SUCCESS Successfully read the admin submission queue base address register content.
272 @return EFI_DEVICE_ERROR Fail to read the admin submission queue base address register.
276 ReadNvmeAdminSubmissionQueueBaseAddress (
277 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
,
281 EFI_PCI_IO_PROTOCOL
*PciIo
;
285 PciIo
= Private
->PciIo
;
286 Status
= PciIo
->Mem
.Read (
295 if (EFI_ERROR(Status
)) {
299 WriteUnaligned64 ((UINT64
*)Asq
, Data
);
304 Write Nvm Express admin submission queue base address register.
306 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
307 @param Asq The buffer used to store the content to be written into admin submission queue base address register.
309 @return EFI_SUCCESS Successfully write data into the admin submission queue base address register.
310 @return EFI_DEVICE_ERROR Fail to write data into the admin submission queue base address register.
314 WriteNvmeAdminSubmissionQueueBaseAddress (
315 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
,
319 EFI_PCI_IO_PROTOCOL
*PciIo
;
323 PciIo
= Private
->PciIo
;
324 Data
= ReadUnaligned64 ((UINT64
*)Asq
);
326 Status
= PciIo
->Mem
.Write (
335 if (EFI_ERROR(Status
)) {
339 DEBUG ((EFI_D_INFO
, "Asq: %lx\n", *Asq
));
345 Read Nvm Express admin completion queue base address register.
347 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
348 @param Acq The buffer used to store admin completion queue base address register content.
350 @return EFI_SUCCESS Successfully read the admin completion queue base address register content.
351 @return EFI_DEVICE_ERROR Fail to read the admin completion queue base address register.
355 ReadNvmeAdminCompletionQueueBaseAddress (
356 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
,
360 EFI_PCI_IO_PROTOCOL
*PciIo
;
364 PciIo
= Private
->PciIo
;
366 Status
= PciIo
->Mem
.Read (
375 if (EFI_ERROR(Status
)) {
379 WriteUnaligned64 ((UINT64
*)Acq
, Data
);
384 Write Nvm Express admin completion queue base address register.
386 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
387 @param Acq The buffer used to store the content to be written into admin completion queue base address register.
389 @return EFI_SUCCESS Successfully write data into the admin completion queue base address register.
390 @return EFI_DEVICE_ERROR Fail to write data into the admin completion queue base address register.
394 WriteNvmeAdminCompletionQueueBaseAddress (
395 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
,
399 EFI_PCI_IO_PROTOCOL
*PciIo
;
403 PciIo
= Private
->PciIo
;
404 Data
= ReadUnaligned64 ((UINT64
*)Acq
);
406 Status
= PciIo
->Mem
.Write (
415 if (EFI_ERROR(Status
)) {
419 DEBUG ((EFI_D_INFO
, "Acq: %lxh\n", *Acq
));
425 Disable the Nvm Express controller.
427 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
429 @return EFI_SUCCESS Successfully disable the controller.
430 @return EFI_DEVICE_ERROR Fail to disable the controller.
434 NvmeDisableController (
435 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
445 // Read Controller Configuration Register.
447 Status
= ReadNvmeControllerConfiguration (Private
, &Cc
);
448 if (EFI_ERROR(Status
)) {
455 // Disable the controller.
457 Status
= WriteNvmeControllerConfiguration (Private
, &Cc
);
459 if (EFI_ERROR(Status
)) {
464 // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to transition from 1 to 0 after
465 // Cc.Enable transition from 1 to 0. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To.
467 if (Private
->Cap
.To
== 0) {
470 Timeout
= Private
->Cap
.To
;
473 for(Index
= (Timeout
* 500); Index
!= 0; --Index
) {
477 // Check if the controller is initialized
479 Status
= ReadNvmeControllerStatus (Private
, &Csts
);
481 if (EFI_ERROR(Status
)) {
491 Status
= EFI_DEVICE_ERROR
;
494 DEBUG ((EFI_D_INFO
, "NVMe controller is disabled with status [%r].\n", Status
));
499 Enable the Nvm Express controller.
501 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
503 @return EFI_SUCCESS Successfully enable the controller.
504 @return EFI_DEVICE_ERROR Fail to enable the controller.
505 @return EFI_TIMEOUT Fail to enable the controller in given time slot.
509 NvmeEnableController (
510 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
520 // Enable the controller.
521 // CC.AMS, CC.MPS and CC.CSS are all set to 0.
523 ZeroMem (&Cc
, sizeof (NVME_CC
));
528 Status
= WriteNvmeControllerConfiguration (Private
, &Cc
);
529 if (EFI_ERROR(Status
)) {
534 // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to set after
535 // Cc.Enable. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To.
537 if (Private
->Cap
.To
== 0) {
540 Timeout
= Private
->Cap
.To
;
543 for(Index
= (Timeout
* 500); Index
!= 0; --Index
) {
547 // Check if the controller is initialized
549 Status
= ReadNvmeControllerStatus (Private
, &Csts
);
551 if (EFI_ERROR(Status
)) {
561 Status
= EFI_TIMEOUT
;
564 DEBUG ((EFI_D_INFO
, "NVMe controller is enabled with status [%r].\n", Status
));
569 Get identify controller data.
571 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
572 @param Buffer The buffer used to store the identify controller data.
574 @return EFI_SUCCESS Successfully get the identify controller data.
575 @return EFI_DEVICE_ERROR Fail to get the identify controller data.
579 NvmeIdentifyController (
580 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
,
584 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket
;
585 EFI_NVM_EXPRESS_COMMAND Command
;
586 EFI_NVM_EXPRESS_COMPLETION Completion
;
589 ZeroMem (&CommandPacket
, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET
));
590 ZeroMem (&Command
, sizeof(EFI_NVM_EXPRESS_COMMAND
));
591 ZeroMem (&Completion
, sizeof(EFI_NVM_EXPRESS_COMPLETION
));
593 Command
.Cdw0
.Opcode
= NVME_ADMIN_IDENTIFY_CMD
;
595 // According to Nvm Express 1.1 spec Figure 38, When not used, the field shall be cleared to 0h.
596 // For the Identify command, the Namespace Identifier is only used for the Namespace data structure.
600 CommandPacket
.NvmeCmd
= &Command
;
601 CommandPacket
.NvmeCompletion
= &Completion
;
602 CommandPacket
.TransferBuffer
= Buffer
;
603 CommandPacket
.TransferLength
= sizeof (NVME_ADMIN_CONTROLLER_DATA
);
604 CommandPacket
.CommandTimeout
= NVME_GENERIC_TIMEOUT
;
605 CommandPacket
.QueueType
= NVME_ADMIN_QUEUE
;
607 // Set bit 0 (Cns bit) to 1 to identify a controller
610 Command
.Flags
= CDW10_VALID
;
612 Status
= Private
->Passthru
.PassThru (
623 Get specified identify namespace data.
625 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
626 @param NamespaceId The specified namespace identifier.
627 @param Buffer The buffer used to store the identify namespace data.
629 @return EFI_SUCCESS Successfully get the identify namespace data.
630 @return EFI_DEVICE_ERROR Fail to get the identify namespace data.
634 NvmeIdentifyNamespace (
635 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
,
636 IN UINT32 NamespaceId
,
640 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket
;
641 EFI_NVM_EXPRESS_COMMAND Command
;
642 EFI_NVM_EXPRESS_COMPLETION Completion
;
645 ZeroMem (&CommandPacket
, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET
));
646 ZeroMem (&Command
, sizeof(EFI_NVM_EXPRESS_COMMAND
));
647 ZeroMem (&Completion
, sizeof(EFI_NVM_EXPRESS_COMPLETION
));
649 CommandPacket
.NvmeCmd
= &Command
;
650 CommandPacket
.NvmeCompletion
= &Completion
;
652 Command
.Cdw0
.Opcode
= NVME_ADMIN_IDENTIFY_CMD
;
653 Command
.Nsid
= NamespaceId
;
654 CommandPacket
.TransferBuffer
= Buffer
;
655 CommandPacket
.TransferLength
= sizeof (NVME_ADMIN_NAMESPACE_DATA
);
656 CommandPacket
.CommandTimeout
= NVME_GENERIC_TIMEOUT
;
657 CommandPacket
.QueueType
= NVME_ADMIN_QUEUE
;
659 // Set bit 0 (Cns bit) to 1 to identify a namespace
661 CommandPacket
.NvmeCmd
->Cdw10
= 0;
662 CommandPacket
.NvmeCmd
->Flags
= CDW10_VALID
;
664 Status
= Private
->Passthru
.PassThru (
675 Create io completion queue.
677 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
679 @return EFI_SUCCESS Successfully create io completion queue.
680 @return EFI_DEVICE_ERROR Fail to create io completion queue.
684 NvmeCreateIoCompletionQueue (
685 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
688 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket
;
689 EFI_NVM_EXPRESS_COMMAND Command
;
690 EFI_NVM_EXPRESS_COMPLETION Completion
;
692 NVME_ADMIN_CRIOCQ CrIoCq
;
696 Status
= EFI_SUCCESS
;
698 for (Index
= 1; Index
< NVME_MAX_QUEUES
; Index
++) {
699 ZeroMem (&CommandPacket
, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET
));
700 ZeroMem (&Command
, sizeof(EFI_NVM_EXPRESS_COMMAND
));
701 ZeroMem (&Completion
, sizeof(EFI_NVM_EXPRESS_COMPLETION
));
702 ZeroMem (&CrIoCq
, sizeof(NVME_ADMIN_CRIOCQ
));
704 CommandPacket
.NvmeCmd
= &Command
;
705 CommandPacket
.NvmeCompletion
= &Completion
;
707 Command
.Cdw0
.Opcode
= NVME_ADMIN_CRIOCQ_CMD
;
708 CommandPacket
.TransferBuffer
= Private
->CqBufferPciAddr
[Index
];
709 CommandPacket
.TransferLength
= EFI_PAGE_SIZE
;
710 CommandPacket
.CommandTimeout
= NVME_GENERIC_TIMEOUT
;
711 CommandPacket
.QueueType
= NVME_ADMIN_QUEUE
;
714 QueueSize
= NVME_CCQ_SIZE
;
716 if (Private
->Cap
.Mqes
> NVME_ASYNC_CCQ_SIZE
) {
717 QueueSize
= NVME_ASYNC_CCQ_SIZE
;
719 QueueSize
= Private
->Cap
.Mqes
;
724 CrIoCq
.Qsize
= QueueSize
;
726 CopyMem (&CommandPacket
.NvmeCmd
->Cdw10
, &CrIoCq
, sizeof (NVME_ADMIN_CRIOCQ
));
727 CommandPacket
.NvmeCmd
->Flags
= CDW10_VALID
| CDW11_VALID
;
729 Status
= Private
->Passthru
.PassThru (
735 if (EFI_ERROR (Status
)) {
744 Create io submission queue.
746 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
748 @return EFI_SUCCESS Successfully create io submission queue.
749 @return EFI_DEVICE_ERROR Fail to create io submission queue.
753 NvmeCreateIoSubmissionQueue (
754 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
757 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket
;
758 EFI_NVM_EXPRESS_COMMAND Command
;
759 EFI_NVM_EXPRESS_COMPLETION Completion
;
761 NVME_ADMIN_CRIOSQ CrIoSq
;
765 Status
= EFI_SUCCESS
;
767 for (Index
= 1; Index
< NVME_MAX_QUEUES
; Index
++) {
768 ZeroMem (&CommandPacket
, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET
));
769 ZeroMem (&Command
, sizeof(EFI_NVM_EXPRESS_COMMAND
));
770 ZeroMem (&Completion
, sizeof(EFI_NVM_EXPRESS_COMPLETION
));
771 ZeroMem (&CrIoSq
, sizeof(NVME_ADMIN_CRIOSQ
));
773 CommandPacket
.NvmeCmd
= &Command
;
774 CommandPacket
.NvmeCompletion
= &Completion
;
776 Command
.Cdw0
.Opcode
= NVME_ADMIN_CRIOSQ_CMD
;
777 CommandPacket
.TransferBuffer
= Private
->SqBufferPciAddr
[Index
];
778 CommandPacket
.TransferLength
= EFI_PAGE_SIZE
;
779 CommandPacket
.CommandTimeout
= NVME_GENERIC_TIMEOUT
;
780 CommandPacket
.QueueType
= NVME_ADMIN_QUEUE
;
783 QueueSize
= NVME_CSQ_SIZE
;
785 if (Private
->Cap
.Mqes
> NVME_ASYNC_CSQ_SIZE
) {
786 QueueSize
= NVME_ASYNC_CSQ_SIZE
;
788 QueueSize
= Private
->Cap
.Mqes
;
793 CrIoSq
.Qsize
= QueueSize
;
797 CopyMem (&CommandPacket
.NvmeCmd
->Cdw10
, &CrIoSq
, sizeof (NVME_ADMIN_CRIOSQ
));
798 CommandPacket
.NvmeCmd
->Flags
= CDW10_VALID
| CDW11_VALID
;
800 Status
= Private
->Passthru
.PassThru (
806 if (EFI_ERROR (Status
)) {
815 Initialize the Nvm Express controller.
817 @param[in] Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
819 @retval EFI_SUCCESS The NVM Express Controller is initialized successfully.
820 @retval Others A device error occurred while initializing the controller.
825 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
829 EFI_PCI_IO_PROTOCOL
*PciIo
;
837 // Save original PCI attributes and enable this controller.
839 PciIo
= Private
->PciIo
;
840 Status
= PciIo
->Attributes (
842 EfiPciIoAttributeOperationGet
,
844 &Private
->PciAttributes
847 if (EFI_ERROR (Status
)) {
851 Status
= PciIo
->Attributes (
853 EfiPciIoAttributeOperationSupported
,
858 if (!EFI_ERROR (Status
)) {
859 Supports
&= (UINT64
)EFI_PCI_DEVICE_ENABLE
;
860 Status
= PciIo
->Attributes (
862 EfiPciIoAttributeOperationEnable
,
868 if (EFI_ERROR (Status
)) {
869 DEBUG ((EFI_D_INFO
, "NvmeControllerInit: failed to enable controller\n"));
874 // Enable 64-bit DMA support in the PCI layer.
876 Status
= PciIo
->Attributes (
878 EfiPciIoAttributeOperationEnable
,
879 EFI_PCI_IO_ATTRIBUTE_DUAL_ADDRESS_CYCLE
,
882 if (EFI_ERROR (Status
)) {
883 DEBUG ((EFI_D_WARN
, "NvmeControllerInit: failed to enable 64-bit DMA (%r)\n", Status
));
887 // Read the Controller Capabilities register and verify that the NVM command set is supported
889 Status
= ReadNvmeControllerCapabilities (Private
, &Private
->Cap
);
890 if (EFI_ERROR (Status
)) {
894 if (Private
->Cap
.Css
!= 0x01) {
895 DEBUG ((EFI_D_INFO
, "NvmeControllerInit: the controller doesn't support NVMe command set\n"));
896 return EFI_UNSUPPORTED
;
900 // Currently the driver only supports 4k page size.
902 ASSERT ((Private
->Cap
.Mpsmin
+ 12) <= EFI_PAGE_SHIFT
);
910 Private
->SqTdbl
[0].Sqt
= 0;
911 Private
->SqTdbl
[1].Sqt
= 0;
912 Private
->SqTdbl
[2].Sqt
= 0;
913 Private
->CqHdbl
[0].Cqh
= 0;
914 Private
->CqHdbl
[1].Cqh
= 0;
915 Private
->CqHdbl
[2].Cqh
= 0;
916 Private
->AsyncSqHead
= 0;
918 Status
= NvmeDisableController (Private
);
920 if (EFI_ERROR(Status
)) {
925 // set number of entries admin submission & completion queues.
927 Aqa
.Asqs
= NVME_ASQ_SIZE
;
929 Aqa
.Acqs
= NVME_ACQ_SIZE
;
933 // Address of admin submission queue.
935 Asq
= (UINT64
)(UINTN
)(Private
->BufferPciAddr
) & ~0xFFF;
938 // Address of admin completion queue.
940 Acq
= (UINT64
)(UINTN
)(Private
->BufferPciAddr
+ EFI_PAGE_SIZE
) & ~0xFFF;
943 // Address of I/O submission & completion queue.
945 ZeroMem (Private
->Buffer
, EFI_PAGES_TO_SIZE (6));
946 Private
->SqBuffer
[0] = (NVME_SQ
*)(UINTN
)(Private
->Buffer
);
947 Private
->SqBufferPciAddr
[0] = (NVME_SQ
*)(UINTN
)(Private
->BufferPciAddr
);
948 Private
->CqBuffer
[0] = (NVME_CQ
*)(UINTN
)(Private
->Buffer
+ 1 * EFI_PAGE_SIZE
);
949 Private
->CqBufferPciAddr
[0] = (NVME_CQ
*)(UINTN
)(Private
->BufferPciAddr
+ 1 * EFI_PAGE_SIZE
);
950 Private
->SqBuffer
[1] = (NVME_SQ
*)(UINTN
)(Private
->Buffer
+ 2 * EFI_PAGE_SIZE
);
951 Private
->SqBufferPciAddr
[1] = (NVME_SQ
*)(UINTN
)(Private
->BufferPciAddr
+ 2 * EFI_PAGE_SIZE
);
952 Private
->CqBuffer
[1] = (NVME_CQ
*)(UINTN
)(Private
->Buffer
+ 3 * EFI_PAGE_SIZE
);
953 Private
->CqBufferPciAddr
[1] = (NVME_CQ
*)(UINTN
)(Private
->BufferPciAddr
+ 3 * EFI_PAGE_SIZE
);
954 Private
->SqBuffer
[2] = (NVME_SQ
*)(UINTN
)(Private
->Buffer
+ 4 * EFI_PAGE_SIZE
);
955 Private
->SqBufferPciAddr
[2] = (NVME_SQ
*)(UINTN
)(Private
->BufferPciAddr
+ 4 * EFI_PAGE_SIZE
);
956 Private
->CqBuffer
[2] = (NVME_CQ
*)(UINTN
)(Private
->Buffer
+ 5 * EFI_PAGE_SIZE
);
957 Private
->CqBufferPciAddr
[2] = (NVME_CQ
*)(UINTN
)(Private
->BufferPciAddr
+ 5 * EFI_PAGE_SIZE
);
959 DEBUG ((EFI_D_INFO
, "Private->Buffer = [%016X]\n", (UINT64
)(UINTN
)Private
->Buffer
));
960 DEBUG ((EFI_D_INFO
, "Admin Submission Queue size (Aqa.Asqs) = [%08X]\n", Aqa
.Asqs
));
961 DEBUG ((EFI_D_INFO
, "Admin Completion Queue size (Aqa.Acqs) = [%08X]\n", Aqa
.Acqs
));
962 DEBUG ((EFI_D_INFO
, "Admin Submission Queue (SqBuffer[0]) = [%016X]\n", Private
->SqBuffer
[0]));
963 DEBUG ((EFI_D_INFO
, "Admin Completion Queue (CqBuffer[0]) = [%016X]\n", Private
->CqBuffer
[0]));
964 DEBUG ((EFI_D_INFO
, "Sync I/O Submission Queue (SqBuffer[1]) = [%016X]\n", Private
->SqBuffer
[1]));
965 DEBUG ((EFI_D_INFO
, "Sync I/O Completion Queue (CqBuffer[1]) = [%016X]\n", Private
->CqBuffer
[1]));
966 DEBUG ((EFI_D_INFO
, "Async I/O Submission Queue (SqBuffer[2]) = [%016X]\n", Private
->SqBuffer
[2]));
967 DEBUG ((EFI_D_INFO
, "Async I/O Completion Queue (CqBuffer[2]) = [%016X]\n", Private
->CqBuffer
[2]));
970 // Program admin queue attributes.
972 Status
= WriteNvmeAdminQueueAttributes (Private
, &Aqa
);
974 if (EFI_ERROR(Status
)) {
979 // Program admin submission queue address.
981 Status
= WriteNvmeAdminSubmissionQueueBaseAddress (Private
, &Asq
);
983 if (EFI_ERROR(Status
)) {
988 // Program admin completion queue address.
990 Status
= WriteNvmeAdminCompletionQueueBaseAddress (Private
, &Acq
);
992 if (EFI_ERROR(Status
)) {
996 Status
= NvmeEnableController (Private
);
997 if (EFI_ERROR(Status
)) {
1002 // Allocate buffer for Identify Controller data
1004 if (Private
->ControllerData
== NULL
) {
1005 Private
->ControllerData
= (NVME_ADMIN_CONTROLLER_DATA
*)AllocateZeroPool (sizeof(NVME_ADMIN_CONTROLLER_DATA
));
1007 if (Private
->ControllerData
== NULL
) {
1008 return EFI_OUT_OF_RESOURCES
;
1013 // Get current Identify Controller Data
1015 Status
= NvmeIdentifyController (Private
, Private
->ControllerData
);
1017 if (EFI_ERROR(Status
)) {
1018 FreePool(Private
->ControllerData
);
1019 Private
->ControllerData
= NULL
;
1020 return EFI_NOT_FOUND
;
1024 // Dump NvmExpress Identify Controller Data
1026 CopyMem (Sn
, Private
->ControllerData
->Sn
, sizeof (Private
->ControllerData
->Sn
));
1028 CopyMem (Mn
, Private
->ControllerData
->Mn
, sizeof (Private
->ControllerData
->Mn
));
1030 DEBUG ((EFI_D_INFO
, " == NVME IDENTIFY CONTROLLER DATA ==\n"));
1031 DEBUG ((EFI_D_INFO
, " PCI VID : 0x%x\n", Private
->ControllerData
->Vid
));
1032 DEBUG ((EFI_D_INFO
, " PCI SSVID : 0x%x\n", Private
->ControllerData
->Ssvid
));
1033 DEBUG ((EFI_D_INFO
, " SN : %a\n", Sn
));
1034 DEBUG ((EFI_D_INFO
, " MN : %a\n", Mn
));
1035 DEBUG ((EFI_D_INFO
, " FR : 0x%x\n", *((UINT64
*)Private
->ControllerData
->Fr
)));
1036 DEBUG ((EFI_D_INFO
, " RAB : 0x%x\n", Private
->ControllerData
->Rab
));
1037 DEBUG ((EFI_D_INFO
, " IEEE : 0x%x\n", *(UINT32
*)Private
->ControllerData
->Ieee_oui
));
1038 DEBUG ((EFI_D_INFO
, " AERL : 0x%x\n", Private
->ControllerData
->Aerl
));
1039 DEBUG ((EFI_D_INFO
, " SQES : 0x%x\n", Private
->ControllerData
->Sqes
));
1040 DEBUG ((EFI_D_INFO
, " CQES : 0x%x\n", Private
->ControllerData
->Cqes
));
1041 DEBUG ((EFI_D_INFO
, " NN : 0x%x\n", Private
->ControllerData
->Nn
));
1044 // Create two I/O completion queues.
1045 // One for blocking I/O, one for non-blocking I/O.
1047 Status
= NvmeCreateIoCompletionQueue (Private
);
1048 if (EFI_ERROR(Status
)) {
1053 // Create two I/O Submission queues.
1054 // One for blocking I/O, one for non-blocking I/O.
1056 Status
= NvmeCreateIoSubmissionQueue (Private
);
1062 This routine is called to properly shutdown the Nvm Express controller per NVMe spec.
1064 @param[in] ResetType The type of reset to perform.
1065 @param[in] ResetStatus The status code for the reset.
1066 @param[in] DataSize The size, in bytes, of ResetData.
1067 @param[in] ResetData For a ResetType of EfiResetCold, EfiResetWarm, or
1068 EfiResetShutdown the data buffer starts with a Null-terminated
1069 string, optionally followed by additional binary data.
1070 The string is a description that the caller may use to further
1071 indicate the reason for the system reset. ResetData is only
1072 valid if ResetStatus is something other than EFI_SUCCESS
1073 unless the ResetType is EfiResetPlatformSpecific
1074 where a minimum amount of ResetData is always required.
1075 For a ResetType of EfiResetPlatformSpecific the data buffer
1076 also starts with a Null-terminated string that is followed
1077 by an EFI_GUID that describes the specific type of reset to perform.
1081 NvmeShutdownAllControllers (
1082 IN EFI_RESET_TYPE ResetType
,
1083 IN EFI_STATUS ResetStatus
,
1085 IN VOID
*ResetData OPTIONAL
1089 EFI_HANDLE
*Handles
;
1092 EFI_OPEN_PROTOCOL_INFORMATION_ENTRY
*OpenInfos
;
1093 UINTN OpenInfoCount
;
1094 UINTN OpenInfoIndex
;
1095 EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL
*NvmePassThru
;
1099 NVME_CONTROLLER_PRIVATE_DATA
*Private
;
1101 Status
= gBS
->LocateHandleBuffer (
1103 &gEfiPciIoProtocolGuid
,
1108 if (EFI_ERROR (Status
)) {
1112 for (HandleIndex
= 0; HandleIndex
< HandleCount
; HandleIndex
++) {
1113 Status
= gBS
->OpenProtocolInformation (
1114 Handles
[HandleIndex
],
1115 &gEfiPciIoProtocolGuid
,
1119 if (EFI_ERROR (Status
)) {
1123 for (OpenInfoIndex
= 0; OpenInfoIndex
< OpenInfoCount
; OpenInfoIndex
++) {
1125 // Find all the NVME controller managed by this driver.
1126 // gImageHandle equals to DriverBinding handle for this driver.
1128 if (((OpenInfos
[OpenInfoIndex
].Attributes
& EFI_OPEN_PROTOCOL_BY_DRIVER
) != 0) &&
1129 (OpenInfos
[OpenInfoIndex
].AgentHandle
== gImageHandle
)) {
1130 Status
= gBS
->OpenProtocol (
1131 OpenInfos
[OpenInfoIndex
].ControllerHandle
,
1132 &gEfiNvmExpressPassThruProtocolGuid
,
1133 (VOID
**) &NvmePassThru
,
1136 EFI_OPEN_PROTOCOL_GET_PROTOCOL
1138 if (EFI_ERROR (Status
)) {
1141 Private
= NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (NvmePassThru
);
1144 // Read Controller Configuration Register.
1146 Status
= ReadNvmeControllerConfiguration (Private
, &Cc
);
1147 if (EFI_ERROR(Status
)) {
1151 // The host should set the Shutdown Notification (CC.SHN) field to 01b
1152 // to indicate a normal shutdown operation.
1154 Cc
.Shn
= NVME_CC_SHN_NORMAL_SHUTDOWN
;
1155 Status
= WriteNvmeControllerConfiguration (Private
, &Cc
);
1156 if (EFI_ERROR(Status
)) {
1161 // The controller indicates when shutdown processing is completed by updating the
1162 // Shutdown Status (CSTS.SHST) field to 10b.
1163 // Wait up to 45 seconds (break down to 4500 x 10ms) for the shutdown to complete.
1165 for (Index
= 0; Index
< NVME_SHUTDOWN_PROCESS_TIMEOUT
* 100; Index
++) {
1166 Status
= ReadNvmeControllerStatus (Private
, &Csts
);
1167 if (!EFI_ERROR(Status
) && (Csts
.Shst
== NVME_CSTS_SHST_SHUTDOWN_COMPLETED
)) {
1168 DEBUG((DEBUG_INFO
, "NvmeShutdownController: shutdown processing is completed after %dms.\n", Index
* 10));
1174 gBS
->Stall (10 * 1000);
1177 if (Index
== NVME_SHUTDOWN_PROCESS_TIMEOUT
* 100) {
1178 DEBUG((DEBUG_ERROR
, "NvmeShutdownController: shutdown processing is timed out\n"));
1186 Register the shutdown notification through the ResetNotification protocol.
1188 Register the shutdown notification when mNvmeControllerNumber increased from 0 to 1.
1191 NvmeRegisterShutdownNotification (
1196 EFI_RESET_NOTIFICATION_PROTOCOL
*ResetNotify
;
1198 mNvmeControllerNumber
++;
1199 if (mNvmeControllerNumber
== 1) {
1200 Status
= gBS
->LocateProtocol (&gEfiResetNotificationProtocolGuid
, NULL
, (VOID
**) &ResetNotify
);
1201 if (!EFI_ERROR (Status
)) {
1202 Status
= ResetNotify
->RegisterResetNotify (ResetNotify
, NvmeShutdownAllControllers
);
1203 ASSERT_EFI_ERROR (Status
);
1205 DEBUG ((DEBUG_WARN
, "NVME: ResetNotification absent! Shutdown notification cannot be performed!\n"));
1211 Unregister the shutdown notification through the ResetNotification protocol.
1213 Unregister the shutdown notification when mNvmeControllerNumber decreased from 1 to 0.
1216 NvmeUnregisterShutdownNotification (
1221 EFI_RESET_NOTIFICATION_PROTOCOL
*ResetNotify
;
1223 mNvmeControllerNumber
--;
1224 if (mNvmeControllerNumber
== 0) {
1225 Status
= gBS
->LocateProtocol (&gEfiResetNotificationProtocolGuid
, NULL
, (VOID
**) &ResetNotify
);
1226 if (!EFI_ERROR (Status
)) {
1227 Status
= ResetNotify
->UnregisterResetNotify (ResetNotify
, NvmeShutdownAllControllers
);
1228 ASSERT_EFI_ERROR (Status
);