2 NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows
3 NVM Express specification.
5 Copyright (c) 2013 - 2016, Intel Corporation. All rights reserved.<BR>
6 This program and the accompanying materials
7 are licensed and made available under the terms and conditions of the BSD License
8 which accompanies this distribution. The full text of the license may be found at
9 http://opensource.org/licenses/bsd-license.php.
11 THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
12 WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
16 #include "NvmExpress.h"
19 Read Nvm Express controller capability register.
21 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
22 @param Cap The buffer used to store capability register content.
24 @return EFI_SUCCESS Successfully read the controller capability register content.
25 @return EFI_DEVICE_ERROR Fail to read the controller capability register.
29 ReadNvmeControllerCapabilities (
30 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
,
34 EFI_PCI_IO_PROTOCOL
*PciIo
;
38 PciIo
= Private
->PciIo
;
39 Status
= PciIo
->Mem
.Read (
48 if (EFI_ERROR(Status
)) {
52 WriteUnaligned64 ((UINT64
*)Cap
, Data
);
57 Read Nvm Express controller configuration register.
59 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
60 @param Cc The buffer used to store configuration register content.
62 @return EFI_SUCCESS Successfully read the controller configuration register content.
63 @return EFI_DEVICE_ERROR Fail to read the controller configuration register.
67 ReadNvmeControllerConfiguration (
68 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
,
72 EFI_PCI_IO_PROTOCOL
*PciIo
;
76 PciIo
= Private
->PciIo
;
77 Status
= PciIo
->Mem
.Read (
86 if (EFI_ERROR(Status
)) {
90 WriteUnaligned32 ((UINT32
*)Cc
, Data
);
95 Write Nvm Express controller configuration register.
97 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
98 @param Cc The buffer used to store the content to be written into configuration register.
100 @return EFI_SUCCESS Successfully write data into the controller configuration register.
101 @return EFI_DEVICE_ERROR Fail to write data into the controller configuration register.
105 WriteNvmeControllerConfiguration (
106 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
,
110 EFI_PCI_IO_PROTOCOL
*PciIo
;
114 PciIo
= Private
->PciIo
;
115 Data
= ReadUnaligned32 ((UINT32
*)Cc
);
116 Status
= PciIo
->Mem
.Write (
125 if (EFI_ERROR(Status
)) {
129 DEBUG ((EFI_D_INFO
, "Cc.En: %d\n", Cc
->En
));
130 DEBUG ((EFI_D_INFO
, "Cc.Css: %d\n", Cc
->Css
));
131 DEBUG ((EFI_D_INFO
, "Cc.Mps: %d\n", Cc
->Mps
));
132 DEBUG ((EFI_D_INFO
, "Cc.Ams: %d\n", Cc
->Ams
));
133 DEBUG ((EFI_D_INFO
, "Cc.Shn: %d\n", Cc
->Shn
));
134 DEBUG ((EFI_D_INFO
, "Cc.Iosqes: %d\n", Cc
->Iosqes
));
135 DEBUG ((EFI_D_INFO
, "Cc.Iocqes: %d\n", Cc
->Iocqes
));
141 Read Nvm Express controller status register.
143 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
144 @param Csts The buffer used to store status register content.
146 @return EFI_SUCCESS Successfully read the controller status register content.
147 @return EFI_DEVICE_ERROR Fail to read the controller status register.
151 ReadNvmeControllerStatus (
152 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
,
156 EFI_PCI_IO_PROTOCOL
*PciIo
;
160 PciIo
= Private
->PciIo
;
161 Status
= PciIo
->Mem
.Read (
170 if (EFI_ERROR(Status
)) {
174 WriteUnaligned32 ((UINT32
*)Csts
, Data
);
179 Read Nvm Express admin queue attributes register.
181 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
182 @param Aqa The buffer used to store admin queue attributes register content.
184 @return EFI_SUCCESS Successfully read the admin queue attributes register content.
185 @return EFI_DEVICE_ERROR Fail to read the admin queue attributes register.
189 ReadNvmeAdminQueueAttributes (
190 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
,
194 EFI_PCI_IO_PROTOCOL
*PciIo
;
198 PciIo
= Private
->PciIo
;
199 Status
= PciIo
->Mem
.Read (
208 if (EFI_ERROR(Status
)) {
212 WriteUnaligned32 ((UINT32
*)Aqa
, Data
);
217 Write Nvm Express admin queue attributes register.
219 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
220 @param Aqa The buffer used to store the content to be written into admin queue attributes register.
222 @return EFI_SUCCESS Successfully write data into the admin queue attributes register.
223 @return EFI_DEVICE_ERROR Fail to write data into the admin queue attributes register.
227 WriteNvmeAdminQueueAttributes (
228 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
,
232 EFI_PCI_IO_PROTOCOL
*PciIo
;
236 PciIo
= Private
->PciIo
;
237 Data
= ReadUnaligned32 ((UINT32
*)Aqa
);
238 Status
= PciIo
->Mem
.Write (
247 if (EFI_ERROR(Status
)) {
251 DEBUG ((EFI_D_INFO
, "Aqa.Asqs: %d\n", Aqa
->Asqs
));
252 DEBUG ((EFI_D_INFO
, "Aqa.Acqs: %d\n", Aqa
->Acqs
));
258 Read Nvm Express admin submission queue base address register.
260 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
261 @param Asq The buffer used to store admin submission queue base address register content.
263 @return EFI_SUCCESS Successfully read the admin submission queue base address register content.
264 @return EFI_DEVICE_ERROR Fail to read the admin submission queue base address register.
268 ReadNvmeAdminSubmissionQueueBaseAddress (
269 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
,
273 EFI_PCI_IO_PROTOCOL
*PciIo
;
277 PciIo
= Private
->PciIo
;
278 Status
= PciIo
->Mem
.Read (
287 if (EFI_ERROR(Status
)) {
291 WriteUnaligned64 ((UINT64
*)Asq
, Data
);
296 Write Nvm Express admin submission queue base address register.
298 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
299 @param Asq The buffer used to store the content to be written into admin submission queue base address register.
301 @return EFI_SUCCESS Successfully write data into the admin submission queue base address register.
302 @return EFI_DEVICE_ERROR Fail to write data into the admin submission queue base address register.
306 WriteNvmeAdminSubmissionQueueBaseAddress (
307 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
,
311 EFI_PCI_IO_PROTOCOL
*PciIo
;
315 PciIo
= Private
->PciIo
;
316 Data
= ReadUnaligned64 ((UINT64
*)Asq
);
318 Status
= PciIo
->Mem
.Write (
327 if (EFI_ERROR(Status
)) {
331 DEBUG ((EFI_D_INFO
, "Asq: %lx\n", *Asq
));
337 Read Nvm Express admin completion queue base address register.
339 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
340 @param Acq The buffer used to store admin completion queue base address register content.
342 @return EFI_SUCCESS Successfully read the admin completion queue base address register content.
343 @return EFI_DEVICE_ERROR Fail to read the admin completion queue base address register.
347 ReadNvmeAdminCompletionQueueBaseAddress (
348 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
,
352 EFI_PCI_IO_PROTOCOL
*PciIo
;
356 PciIo
= Private
->PciIo
;
358 Status
= PciIo
->Mem
.Read (
367 if (EFI_ERROR(Status
)) {
371 WriteUnaligned64 ((UINT64
*)Acq
, Data
);
376 Write Nvm Express admin completion queue base address register.
378 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
379 @param Acq The buffer used to store the content to be written into admin completion queue base address register.
381 @return EFI_SUCCESS Successfully write data into the admin completion queue base address register.
382 @return EFI_DEVICE_ERROR Fail to write data into the admin completion queue base address register.
386 WriteNvmeAdminCompletionQueueBaseAddress (
387 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
,
391 EFI_PCI_IO_PROTOCOL
*PciIo
;
395 PciIo
= Private
->PciIo
;
396 Data
= ReadUnaligned64 ((UINT64
*)Acq
);
398 Status
= PciIo
->Mem
.Write (
407 if (EFI_ERROR(Status
)) {
411 DEBUG ((EFI_D_INFO
, "Acq: %lxh\n", *Acq
));
417 Disable the Nvm Express controller.
419 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
421 @return EFI_SUCCESS Successfully disable the controller.
422 @return EFI_DEVICE_ERROR Fail to disable the controller.
426 NvmeDisableController (
427 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
437 // Read Controller Configuration Register.
439 Status
= ReadNvmeControllerConfiguration (Private
, &Cc
);
440 if (EFI_ERROR(Status
)) {
447 // Disable the controller.
449 Status
= WriteNvmeControllerConfiguration (Private
, &Cc
);
451 if (EFI_ERROR(Status
)) {
456 // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to transition from 1 to 0 after
457 // Cc.Enable transition from 1 to 0. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To.
459 if (Private
->Cap
.To
== 0) {
462 Timeout
= Private
->Cap
.To
;
465 for(Index
= (Timeout
* 500); Index
!= 0; --Index
) {
469 // Check if the controller is initialized
471 Status
= ReadNvmeControllerStatus (Private
, &Csts
);
473 if (EFI_ERROR(Status
)) {
483 Status
= EFI_DEVICE_ERROR
;
486 DEBUG ((EFI_D_INFO
, "NVMe controller is disabled with status [%r].\n", Status
));
491 Enable the Nvm Express controller.
493 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
495 @return EFI_SUCCESS Successfully enable the controller.
496 @return EFI_DEVICE_ERROR Fail to enable the controller.
497 @return EFI_TIMEOUT Fail to enable the controller in given time slot.
501 NvmeEnableController (
502 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
512 // Enable the controller.
513 // CC.AMS, CC.MPS and CC.CSS are all set to 0.
515 ZeroMem (&Cc
, sizeof (NVME_CC
));
520 Status
= WriteNvmeControllerConfiguration (Private
, &Cc
);
521 if (EFI_ERROR(Status
)) {
526 // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to set after
527 // Cc.Enable. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To.
529 if (Private
->Cap
.To
== 0) {
532 Timeout
= Private
->Cap
.To
;
535 for(Index
= (Timeout
* 500); Index
!= 0; --Index
) {
539 // Check if the controller is initialized
541 Status
= ReadNvmeControllerStatus (Private
, &Csts
);
543 if (EFI_ERROR(Status
)) {
553 Status
= EFI_TIMEOUT
;
556 DEBUG ((EFI_D_INFO
, "NVMe controller is enabled with status [%r].\n", Status
));
561 Get identify controller data.
563 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
564 @param Buffer The buffer used to store the identify controller data.
566 @return EFI_SUCCESS Successfully get the identify controller data.
567 @return EFI_DEVICE_ERROR Fail to get the identify controller data.
571 NvmeIdentifyController (
572 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
,
576 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket
;
577 EFI_NVM_EXPRESS_COMMAND Command
;
578 EFI_NVM_EXPRESS_COMPLETION Completion
;
581 ZeroMem (&CommandPacket
, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET
));
582 ZeroMem (&Command
, sizeof(EFI_NVM_EXPRESS_COMMAND
));
583 ZeroMem (&Completion
, sizeof(EFI_NVM_EXPRESS_COMPLETION
));
585 Command
.Cdw0
.Opcode
= NVME_ADMIN_IDENTIFY_CMD
;
587 // According to Nvm Express 1.1 spec Figure 38, When not used, the field shall be cleared to 0h.
588 // For the Identify command, the Namespace Identifier is only used for the Namespace data structure.
592 CommandPacket
.NvmeCmd
= &Command
;
593 CommandPacket
.NvmeCompletion
= &Completion
;
594 CommandPacket
.TransferBuffer
= Buffer
;
595 CommandPacket
.TransferLength
= sizeof (NVME_ADMIN_CONTROLLER_DATA
);
596 CommandPacket
.CommandTimeout
= NVME_GENERIC_TIMEOUT
;
597 CommandPacket
.QueueType
= NVME_ADMIN_QUEUE
;
599 // Set bit 0 (Cns bit) to 1 to identify a controller
602 Command
.Flags
= CDW10_VALID
;
604 Status
= Private
->Passthru
.PassThru (
615 Get specified identify namespace data.
617 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
618 @param NamespaceId The specified namespace identifier.
619 @param Buffer The buffer used to store the identify namespace data.
621 @return EFI_SUCCESS Successfully get the identify namespace data.
622 @return EFI_DEVICE_ERROR Fail to get the identify namespace data.
626 NvmeIdentifyNamespace (
627 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
,
628 IN UINT32 NamespaceId
,
632 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket
;
633 EFI_NVM_EXPRESS_COMMAND Command
;
634 EFI_NVM_EXPRESS_COMPLETION Completion
;
637 ZeroMem (&CommandPacket
, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET
));
638 ZeroMem (&Command
, sizeof(EFI_NVM_EXPRESS_COMMAND
));
639 ZeroMem (&Completion
, sizeof(EFI_NVM_EXPRESS_COMPLETION
));
641 CommandPacket
.NvmeCmd
= &Command
;
642 CommandPacket
.NvmeCompletion
= &Completion
;
644 Command
.Cdw0
.Opcode
= NVME_ADMIN_IDENTIFY_CMD
;
645 Command
.Nsid
= NamespaceId
;
646 CommandPacket
.TransferBuffer
= Buffer
;
647 CommandPacket
.TransferLength
= sizeof (NVME_ADMIN_NAMESPACE_DATA
);
648 CommandPacket
.CommandTimeout
= NVME_GENERIC_TIMEOUT
;
649 CommandPacket
.QueueType
= NVME_ADMIN_QUEUE
;
651 // Set bit 0 (Cns bit) to 1 to identify a namespace
653 CommandPacket
.NvmeCmd
->Cdw10
= 0;
654 CommandPacket
.NvmeCmd
->Flags
= CDW10_VALID
;
656 Status
= Private
->Passthru
.PassThru (
667 Create io completion queue.
669 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
671 @return EFI_SUCCESS Successfully create io completion queue.
672 @return EFI_DEVICE_ERROR Fail to create io completion queue.
676 NvmeCreateIoCompletionQueue (
677 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
680 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket
;
681 EFI_NVM_EXPRESS_COMMAND Command
;
682 EFI_NVM_EXPRESS_COMPLETION Completion
;
684 NVME_ADMIN_CRIOCQ CrIoCq
;
688 Status
= EFI_SUCCESS
;
690 for (Index
= 1; Index
< NVME_MAX_QUEUES
; Index
++) {
691 ZeroMem (&CommandPacket
, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET
));
692 ZeroMem (&Command
, sizeof(EFI_NVM_EXPRESS_COMMAND
));
693 ZeroMem (&Completion
, sizeof(EFI_NVM_EXPRESS_COMPLETION
));
694 ZeroMem (&CrIoCq
, sizeof(NVME_ADMIN_CRIOCQ
));
696 CommandPacket
.NvmeCmd
= &Command
;
697 CommandPacket
.NvmeCompletion
= &Completion
;
699 Command
.Cdw0
.Opcode
= NVME_ADMIN_CRIOCQ_CMD
;
700 CommandPacket
.TransferBuffer
= Private
->CqBufferPciAddr
[Index
];
701 CommandPacket
.TransferLength
= EFI_PAGE_SIZE
;
702 CommandPacket
.CommandTimeout
= NVME_GENERIC_TIMEOUT
;
703 CommandPacket
.QueueType
= NVME_ADMIN_QUEUE
;
706 QueueSize
= NVME_CCQ_SIZE
;
708 if (Private
->Cap
.Mqes
> NVME_ASYNC_CCQ_SIZE
) {
709 QueueSize
= NVME_ASYNC_CCQ_SIZE
;
711 QueueSize
= Private
->Cap
.Mqes
;
716 CrIoCq
.Qsize
= QueueSize
;
718 CopyMem (&CommandPacket
.NvmeCmd
->Cdw10
, &CrIoCq
, sizeof (NVME_ADMIN_CRIOCQ
));
719 CommandPacket
.NvmeCmd
->Flags
= CDW10_VALID
| CDW11_VALID
;
721 Status
= Private
->Passthru
.PassThru (
727 if (EFI_ERROR (Status
)) {
736 Create io submission queue.
738 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
740 @return EFI_SUCCESS Successfully create io submission queue.
741 @return EFI_DEVICE_ERROR Fail to create io submission queue.
745 NvmeCreateIoSubmissionQueue (
746 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
749 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket
;
750 EFI_NVM_EXPRESS_COMMAND Command
;
751 EFI_NVM_EXPRESS_COMPLETION Completion
;
753 NVME_ADMIN_CRIOSQ CrIoSq
;
757 Status
= EFI_SUCCESS
;
759 for (Index
= 1; Index
< NVME_MAX_QUEUES
; Index
++) {
760 ZeroMem (&CommandPacket
, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET
));
761 ZeroMem (&Command
, sizeof(EFI_NVM_EXPRESS_COMMAND
));
762 ZeroMem (&Completion
, sizeof(EFI_NVM_EXPRESS_COMPLETION
));
763 ZeroMem (&CrIoSq
, sizeof(NVME_ADMIN_CRIOSQ
));
765 CommandPacket
.NvmeCmd
= &Command
;
766 CommandPacket
.NvmeCompletion
= &Completion
;
768 Command
.Cdw0
.Opcode
= NVME_ADMIN_CRIOSQ_CMD
;
769 CommandPacket
.TransferBuffer
= Private
->SqBufferPciAddr
[Index
];
770 CommandPacket
.TransferLength
= EFI_PAGE_SIZE
;
771 CommandPacket
.CommandTimeout
= NVME_GENERIC_TIMEOUT
;
772 CommandPacket
.QueueType
= NVME_ADMIN_QUEUE
;
775 QueueSize
= NVME_CSQ_SIZE
;
777 if (Private
->Cap
.Mqes
> NVME_ASYNC_CSQ_SIZE
) {
778 QueueSize
= NVME_ASYNC_CSQ_SIZE
;
780 QueueSize
= Private
->Cap
.Mqes
;
785 CrIoSq
.Qsize
= QueueSize
;
789 CopyMem (&CommandPacket
.NvmeCmd
->Cdw10
, &CrIoSq
, sizeof (NVME_ADMIN_CRIOSQ
));
790 CommandPacket
.NvmeCmd
->Flags
= CDW10_VALID
| CDW11_VALID
;
792 Status
= Private
->Passthru
.PassThru (
798 if (EFI_ERROR (Status
)) {
807 Initialize the Nvm Express controller.
809 @param[in] Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
811 @retval EFI_SUCCESS The NVM Express Controller is initialized successfully.
812 @retval Others A device error occurred while initializing the controller.
817 IN NVME_CONTROLLER_PRIVATE_DATA
*Private
821 EFI_PCI_IO_PROTOCOL
*PciIo
;
829 // Save original PCI attributes and enable this controller.
831 PciIo
= Private
->PciIo
;
832 Status
= PciIo
->Attributes (
834 EfiPciIoAttributeOperationGet
,
836 &Private
->PciAttributes
839 if (EFI_ERROR (Status
)) {
843 Status
= PciIo
->Attributes (
845 EfiPciIoAttributeOperationSupported
,
850 if (!EFI_ERROR (Status
)) {
851 Supports
&= (UINT64
)EFI_PCI_DEVICE_ENABLE
;
852 Status
= PciIo
->Attributes (
854 EfiPciIoAttributeOperationEnable
,
860 if (EFI_ERROR (Status
)) {
861 DEBUG ((EFI_D_INFO
, "NvmeControllerInit: failed to enable controller\n"));
866 // Enable 64-bit DMA support in the PCI layer.
868 Status
= PciIo
->Attributes (
870 EfiPciIoAttributeOperationEnable
,
871 EFI_PCI_IO_ATTRIBUTE_DUAL_ADDRESS_CYCLE
,
874 if (EFI_ERROR (Status
)) {
875 DEBUG ((EFI_D_WARN
, "NvmeControllerInit: failed to enable 64-bit DMA (%r)\n", Status
));
879 // Read the Controller Capabilities register and verify that the NVM command set is supported
881 Status
= ReadNvmeControllerCapabilities (Private
, &Private
->Cap
);
882 if (EFI_ERROR (Status
)) {
886 if (Private
->Cap
.Css
!= 0x01) {
887 DEBUG ((EFI_D_INFO
, "NvmeControllerInit: the controller doesn't support NVMe command set\n"));
888 return EFI_UNSUPPORTED
;
892 // Currently the driver only supports 4k page size.
894 ASSERT ((Private
->Cap
.Mpsmin
+ 12) <= EFI_PAGE_SHIFT
);
902 Private
->SqTdbl
[0].Sqt
= 0;
903 Private
->SqTdbl
[1].Sqt
= 0;
904 Private
->SqTdbl
[2].Sqt
= 0;
905 Private
->CqHdbl
[0].Cqh
= 0;
906 Private
->CqHdbl
[1].Cqh
= 0;
907 Private
->CqHdbl
[2].Cqh
= 0;
908 Private
->AsyncSqHead
= 0;
910 Status
= NvmeDisableController (Private
);
912 if (EFI_ERROR(Status
)) {
917 // set number of entries admin submission & completion queues.
919 Aqa
.Asqs
= NVME_ASQ_SIZE
;
921 Aqa
.Acqs
= NVME_ACQ_SIZE
;
925 // Address of admin submission queue.
927 Asq
= (UINT64
)(UINTN
)(Private
->BufferPciAddr
) & ~0xFFF;
930 // Address of admin completion queue.
932 Acq
= (UINT64
)(UINTN
)(Private
->BufferPciAddr
+ EFI_PAGE_SIZE
) & ~0xFFF;
935 // Address of I/O submission & completion queue.
937 ZeroMem (Private
->Buffer
, EFI_PAGES_TO_SIZE (6));
938 Private
->SqBuffer
[0] = (NVME_SQ
*)(UINTN
)(Private
->Buffer
);
939 Private
->SqBufferPciAddr
[0] = (NVME_SQ
*)(UINTN
)(Private
->BufferPciAddr
);
940 Private
->CqBuffer
[0] = (NVME_CQ
*)(UINTN
)(Private
->Buffer
+ 1 * EFI_PAGE_SIZE
);
941 Private
->CqBufferPciAddr
[0] = (NVME_CQ
*)(UINTN
)(Private
->BufferPciAddr
+ 1 * EFI_PAGE_SIZE
);
942 Private
->SqBuffer
[1] = (NVME_SQ
*)(UINTN
)(Private
->Buffer
+ 2 * EFI_PAGE_SIZE
);
943 Private
->SqBufferPciAddr
[1] = (NVME_SQ
*)(UINTN
)(Private
->BufferPciAddr
+ 2 * EFI_PAGE_SIZE
);
944 Private
->CqBuffer
[1] = (NVME_CQ
*)(UINTN
)(Private
->Buffer
+ 3 * EFI_PAGE_SIZE
);
945 Private
->CqBufferPciAddr
[1] = (NVME_CQ
*)(UINTN
)(Private
->BufferPciAddr
+ 3 * EFI_PAGE_SIZE
);
946 Private
->SqBuffer
[2] = (NVME_SQ
*)(UINTN
)(Private
->Buffer
+ 4 * EFI_PAGE_SIZE
);
947 Private
->SqBufferPciAddr
[2] = (NVME_SQ
*)(UINTN
)(Private
->BufferPciAddr
+ 4 * EFI_PAGE_SIZE
);
948 Private
->CqBuffer
[2] = (NVME_CQ
*)(UINTN
)(Private
->Buffer
+ 5 * EFI_PAGE_SIZE
);
949 Private
->CqBufferPciAddr
[2] = (NVME_CQ
*)(UINTN
)(Private
->BufferPciAddr
+ 5 * EFI_PAGE_SIZE
);
951 DEBUG ((EFI_D_INFO
, "Private->Buffer = [%016X]\n", (UINT64
)(UINTN
)Private
->Buffer
));
952 DEBUG ((EFI_D_INFO
, "Admin Submission Queue size (Aqa.Asqs) = [%08X]\n", Aqa
.Asqs
));
953 DEBUG ((EFI_D_INFO
, "Admin Completion Queue size (Aqa.Acqs) = [%08X]\n", Aqa
.Acqs
));
954 DEBUG ((EFI_D_INFO
, "Admin Submission Queue (SqBuffer[0]) = [%016X]\n", Private
->SqBuffer
[0]));
955 DEBUG ((EFI_D_INFO
, "Admin Completion Queue (CqBuffer[0]) = [%016X]\n", Private
->CqBuffer
[0]));
956 DEBUG ((EFI_D_INFO
, "Sync I/O Submission Queue (SqBuffer[1]) = [%016X]\n", Private
->SqBuffer
[1]));
957 DEBUG ((EFI_D_INFO
, "Sync I/O Completion Queue (CqBuffer[1]) = [%016X]\n", Private
->CqBuffer
[1]));
958 DEBUG ((EFI_D_INFO
, "Async I/O Submission Queue (SqBuffer[2]) = [%016X]\n", Private
->SqBuffer
[2]));
959 DEBUG ((EFI_D_INFO
, "Async I/O Completion Queue (CqBuffer[2]) = [%016X]\n", Private
->CqBuffer
[2]));
962 // Program admin queue attributes.
964 Status
= WriteNvmeAdminQueueAttributes (Private
, &Aqa
);
966 if (EFI_ERROR(Status
)) {
971 // Program admin submission queue address.
973 Status
= WriteNvmeAdminSubmissionQueueBaseAddress (Private
, &Asq
);
975 if (EFI_ERROR(Status
)) {
980 // Program admin completion queue address.
982 Status
= WriteNvmeAdminCompletionQueueBaseAddress (Private
, &Acq
);
984 if (EFI_ERROR(Status
)) {
988 Status
= NvmeEnableController (Private
);
989 if (EFI_ERROR(Status
)) {
994 // Allocate buffer for Identify Controller data
996 if (Private
->ControllerData
== NULL
) {
997 Private
->ControllerData
= (NVME_ADMIN_CONTROLLER_DATA
*)AllocateZeroPool (sizeof(NVME_ADMIN_CONTROLLER_DATA
));
999 if (Private
->ControllerData
== NULL
) {
1000 return EFI_OUT_OF_RESOURCES
;
1005 // Get current Identify Controller Data
1007 Status
= NvmeIdentifyController (Private
, Private
->ControllerData
);
1009 if (EFI_ERROR(Status
)) {
1010 FreePool(Private
->ControllerData
);
1011 Private
->ControllerData
= NULL
;
1012 return EFI_NOT_FOUND
;
1016 // Dump NvmExpress Identify Controller Data
1018 CopyMem (Sn
, Private
->ControllerData
->Sn
, sizeof (Private
->ControllerData
->Sn
));
1020 CopyMem (Mn
, Private
->ControllerData
->Mn
, sizeof (Private
->ControllerData
->Mn
));
1022 DEBUG ((EFI_D_INFO
, " == NVME IDENTIFY CONTROLLER DATA ==\n"));
1023 DEBUG ((EFI_D_INFO
, " PCI VID : 0x%x\n", Private
->ControllerData
->Vid
));
1024 DEBUG ((EFI_D_INFO
, " PCI SSVID : 0x%x\n", Private
->ControllerData
->Ssvid
));
1025 DEBUG ((EFI_D_INFO
, " SN : %a\n", Sn
));
1026 DEBUG ((EFI_D_INFO
, " MN : %a\n", Mn
));
1027 DEBUG ((EFI_D_INFO
, " FR : 0x%x\n", *((UINT64
*)Private
->ControllerData
->Fr
)));
1028 DEBUG ((EFI_D_INFO
, " RAB : 0x%x\n", Private
->ControllerData
->Rab
));
1029 DEBUG ((EFI_D_INFO
, " IEEE : 0x%x\n", *(UINT32
*)Private
->ControllerData
->Ieee_oui
));
1030 DEBUG ((EFI_D_INFO
, " AERL : 0x%x\n", Private
->ControllerData
->Aerl
));
1031 DEBUG ((EFI_D_INFO
, " SQES : 0x%x\n", Private
->ControllerData
->Sqes
));
1032 DEBUG ((EFI_D_INFO
, " CQES : 0x%x\n", Private
->ControllerData
->Cqes
));
1033 DEBUG ((EFI_D_INFO
, " NN : 0x%x\n", Private
->ControllerData
->Nn
));
1036 // Create two I/O completion queues.
1037 // One for blocking I/O, one for non-blocking I/O.
1039 Status
= NvmeCreateIoCompletionQueue (Private
);
1040 if (EFI_ERROR(Status
)) {
1045 // Create two I/O Submission queues.
1046 // One for blocking I/O, one for non-blocking I/O.
1048 Status
= NvmeCreateIoSubmissionQueue (Private
);
1049 if (EFI_ERROR(Status
)) {