2 The NvmExpressPei driver is used to manage non-volatile memory subsystem
3 which follows NVM Express specification at PEI phase.
5 Copyright (c) 2018 - 2019, Intel Corporation. All rights reserved.<BR>
7 SPDX-License-Identifier: BSD-2-Clause-Patent
11 #include "NvmExpressPei.h"
14 Transfer MMIO Data to memory.
16 @param[in,out] MemBuffer Destination: Memory address.
17 @param[in] MmioAddr Source: MMIO address.
18 @param[in] Size Size for read.
20 @retval EFI_SUCCESS MMIO read sucessfully.
25 IN OUT VOID
*MemBuffer
,
34 // priority has adjusted
37 *((UINT32
*)MemBuffer
) = MmioRead32 (MmioAddr
);
41 *((UINT64
*)MemBuffer
) = MmioRead64 (MmioAddr
);
45 *((UINT16
*)MemBuffer
) = MmioRead16 (MmioAddr
);
49 *((UINT8
*)MemBuffer
) = MmioRead8 (MmioAddr
);
53 Ptr
= (UINT8
*)MemBuffer
;
54 for (Offset
= 0; Offset
< Size
; Offset
+= 1) {
55 Data
= MmioRead8 (MmioAddr
+ Offset
);
66 Transfer memory data to MMIO.
68 @param[in,out] MmioAddr Destination: MMIO address.
69 @param[in] MemBuffer Source: Memory address.
70 @param[in] Size Size for write.
72 @retval EFI_SUCCESS MMIO write sucessfully.
77 IN OUT UINTN MmioAddr
,
86 // priority has adjusted
89 MmioWrite32 (MmioAddr
, *((UINT32
*)MemBuffer
));
93 MmioWrite64 (MmioAddr
, *((UINT64
*)MemBuffer
));
97 MmioWrite16 (MmioAddr
, *((UINT16
*)MemBuffer
));
101 MmioWrite8 (MmioAddr
, *((UINT8
*)MemBuffer
));
105 Ptr
= (UINT8
*)MemBuffer
;
106 for (Offset
= 0; Offset
< Size
; Offset
+= 1) {
108 MmioWrite8 (MmioAddr
+ Offset
, Data
);
118 Get the page offset for specific NVME based memory.
120 @param[in] BaseMemIndex The Index of BaseMem (0-based).
122 @retval - The page count for specific BaseMem Index
126 NvmeBaseMemPageOffset (
127 IN UINTN BaseMemIndex
132 UINT32 PageSizeList
[5];
134 PageSizeList
[0] = 1; /* ASQ */
135 PageSizeList
[1] = 1; /* ACQ */
136 PageSizeList
[2] = 1; /* SQs */
137 PageSizeList
[3] = 1; /* CQs */
138 PageSizeList
[4] = NVME_PRP_SIZE
; /* PRPs */
140 if (BaseMemIndex
> MAX_BASEMEM_COUNT
) {
141 DEBUG ((DEBUG_ERROR
, "%a: The input BaseMem index is invalid.\n", __FUNCTION__
));
147 for (Index
= 0; Index
< BaseMemIndex
; Index
++) {
148 Pages
+= PageSizeList
[Index
];
155 Wait for NVME controller status to be ready or not.
157 @param[in] Private The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
158 @param[in] WaitReady Flag for waitting status ready or not.
160 @return EFI_SUCCESS Successfully to wait specific status.
161 @return others Fail to wait for specific controller status.
166 IN PEI_NVME_CONTROLLER_PRIVATE_DATA
*Private
,
176 // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to set after
177 // Cc.Enable. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To.
179 if (Private
->Cap
.To
== 0) {
182 Timeout
= Private
->Cap
.To
;
185 Status
= EFI_SUCCESS
;
186 for (Index
= (Timeout
* 500); Index
!= 0; --Index
) {
187 MicroSecondDelay (1000);
190 // Check if the controller is initialized
192 Status
= NVME_GET_CSTS (Private
, &Csts
);
193 if (EFI_ERROR (Status
)) {
194 DEBUG ((DEBUG_ERROR
, "%a: NVME_GET_CSTS fail, Status - %r\n", __FUNCTION__
, Status
));
198 if ((BOOLEAN
)Csts
.Rdy
== WaitReady
) {
204 Status
= EFI_TIMEOUT
;
211 Disable the Nvm Express controller.
213 @param[in] Private The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
215 @return EFI_SUCCESS Successfully disable the controller.
216 @return others Fail to disable the controller.
220 NvmeDisableController (
221 IN PEI_NVME_CONTROLLER_PRIVATE_DATA
*Private
228 Status
= NVME_GET_CSTS (Private
, &Csts
);
231 // Read Controller Configuration Register.
233 Status
= NVME_GET_CC (Private
, &Cc
);
234 if (EFI_ERROR (Status
)) {
235 DEBUG ((DEBUG_ERROR
, "%a: NVME_GET_CC fail, Status - %r\n", __FUNCTION__
, Status
));
242 // Disable the controller.
244 Status
= NVME_SET_CC (Private
, &Cc
);
245 if (EFI_ERROR (Status
)) {
246 DEBUG ((DEBUG_ERROR
, "%a: NVME_SET_CC fail, Status - %r\n", __FUNCTION__
, Status
));
251 Status
= NvmeWaitController (Private
, FALSE
);
252 if (EFI_ERROR (Status
)) {
253 DEBUG ((DEBUG_ERROR
, "%a: NvmeWaitController fail, Status - %r\n", __FUNCTION__
, Status
));
260 DEBUG ((DEBUG_ERROR
, "%a fail, Status - %r\n", __FUNCTION__
, Status
));
265 Enable the Nvm Express controller.
267 @param[in] Private The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
269 @return EFI_SUCCESS Successfully enable the controller.
270 @return EFI_DEVICE_ERROR Fail to enable the controller.
271 @return EFI_TIMEOUT Fail to enable the controller in given time slot.
275 NvmeEnableController (
276 IN PEI_NVME_CONTROLLER_PRIVATE_DATA
*Private
283 // Enable the controller
284 // CC.AMS, CC.MPS and CC.CSS are all set to 0
286 ZeroMem (&Cc
, sizeof (NVME_CC
));
290 Status
= NVME_SET_CC (Private
, &Cc
);
291 if (EFI_ERROR (Status
)) {
292 DEBUG ((DEBUG_ERROR
, "%a: NVME_SET_CC fail, Status - %r\n", __FUNCTION__
, Status
));
296 Status
= NvmeWaitController (Private
, TRUE
);
297 if (EFI_ERROR (Status
)) {
298 DEBUG ((DEBUG_ERROR
, "%a: NvmeWaitController fail, Status - %r\n", __FUNCTION__
, Status
));
305 DEBUG ((DEBUG_ERROR
, "%a fail, Status: %r\n", __FUNCTION__
, Status
));
310 Get the Identify Controller data.
312 @param[in] Private The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
313 @param[in] Buffer The Buffer used to store the Identify Controller data.
315 @return EFI_SUCCESS Successfully get the Identify Controller data.
316 @return others Fail to get the Identify Controller data.
320 NvmeIdentifyController (
321 IN PEI_NVME_CONTROLLER_PRIVATE_DATA
*Private
,
325 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket
;
326 EFI_NVM_EXPRESS_COMMAND Command
;
327 EFI_NVM_EXPRESS_COMPLETION Completion
;
330 ZeroMem (&CommandPacket
, sizeof (EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET
));
331 ZeroMem (&Command
, sizeof (EFI_NVM_EXPRESS_COMMAND
));
332 ZeroMem (&Completion
, sizeof (EFI_NVM_EXPRESS_COMPLETION
));
334 Command
.Cdw0
.Opcode
= NVME_ADMIN_IDENTIFY_CMD
;
336 // According to Nvm Express 1.1 spec Figure 38, When not used, the field shall be cleared to 0h.
337 // For the Identify command, the Namespace Identifier is only used for the Namespace Data structure.
341 CommandPacket
.NvmeCmd
= &Command
;
342 CommandPacket
.NvmeCompletion
= &Completion
;
343 CommandPacket
.TransferBuffer
= Buffer
;
344 CommandPacket
.TransferLength
= sizeof (NVME_ADMIN_CONTROLLER_DATA
);
345 CommandPacket
.CommandTimeout
= NVME_GENERIC_TIMEOUT
;
346 CommandPacket
.QueueType
= NVME_ADMIN_QUEUE
;
348 // Set bit 0 (Cns bit) to 1 to identify the controller
350 CommandPacket
.NvmeCmd
->Cdw10
= 1;
351 CommandPacket
.NvmeCmd
->Flags
= CDW10_VALID
;
353 Status
= NvmePassThruExecute (
355 NVME_CONTROLLER_NSID
,
362 Get specified identify namespace data.
364 @param[in] Private The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
365 @param[in] NamespaceId The specified namespace identifier.
366 @param[in] Buffer The buffer used to store the identify namespace data.
368 @return EFI_SUCCESS Successfully get the identify namespace data.
369 @return EFI_DEVICE_ERROR Fail to get the identify namespace data.
373 NvmeIdentifyNamespace (
374 IN PEI_NVME_CONTROLLER_PRIVATE_DATA
*Private
,
375 IN UINT32 NamespaceId
,
379 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket
;
380 EFI_NVM_EXPRESS_COMMAND Command
;
381 EFI_NVM_EXPRESS_COMPLETION Completion
;
384 ZeroMem (&CommandPacket
, sizeof (EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET
));
385 ZeroMem (&Command
, sizeof (EFI_NVM_EXPRESS_COMMAND
));
386 ZeroMem (&Completion
, sizeof (EFI_NVM_EXPRESS_COMPLETION
));
388 Command
.Cdw0
.Opcode
= NVME_ADMIN_IDENTIFY_CMD
;
389 Command
.Nsid
= NamespaceId
;
391 CommandPacket
.NvmeCmd
= &Command
;
392 CommandPacket
.NvmeCompletion
= &Completion
;
393 CommandPacket
.TransferBuffer
= Buffer
;
394 CommandPacket
.TransferLength
= sizeof (NVME_ADMIN_NAMESPACE_DATA
);
395 CommandPacket
.CommandTimeout
= NVME_GENERIC_TIMEOUT
;
396 CommandPacket
.QueueType
= NVME_ADMIN_QUEUE
;
398 // Set bit 0 (Cns bit) to 1 to identify a namespace
400 CommandPacket
.NvmeCmd
->Cdw10
= 0;
401 CommandPacket
.NvmeCmd
->Flags
= CDW10_VALID
;
403 Status
= NvmePassThruExecute (
412 Dump the Identify Controller data.
414 @param[in] ControllerData The pointer to the NVME_ADMIN_CONTROLLER_DATA data structure.
418 NvmeDumpControllerData (
419 IN NVME_ADMIN_CONTROLLER_DATA
*ControllerData
425 CopyMem (Sn
, ControllerData
->Sn
, sizeof (ControllerData
->Sn
));
427 CopyMem (Mn
, ControllerData
->Mn
, sizeof (ControllerData
->Mn
));
430 DEBUG ((DEBUG_INFO
, " == NVME IDENTIFY CONTROLLER DATA ==\n"));
431 DEBUG ((DEBUG_INFO
, " PCI VID : 0x%x\n", ControllerData
->Vid
));
432 DEBUG ((DEBUG_INFO
, " PCI SSVID : 0x%x\n", ControllerData
->Ssvid
));
433 DEBUG ((DEBUG_INFO
, " SN : %a\n", Sn
));
434 DEBUG ((DEBUG_INFO
, " MN : %a\n", Mn
));
435 DEBUG ((DEBUG_INFO
, " FR : 0x%lx\n", *((UINT64
*)ControllerData
->Fr
)));
436 DEBUG ((DEBUG_INFO
, " RAB : 0x%x\n", ControllerData
->Rab
));
437 DEBUG ((DEBUG_INFO
, " IEEE : 0x%x\n", *(UINT32
*)ControllerData
->Ieee_oui
));
438 DEBUG ((DEBUG_INFO
, " AERL : 0x%x\n", ControllerData
->Aerl
));
439 DEBUG ((DEBUG_INFO
, " SQES : 0x%x\n", ControllerData
->Sqes
));
440 DEBUG ((DEBUG_INFO
, " CQES : 0x%x\n", ControllerData
->Cqes
));
441 DEBUG ((DEBUG_INFO
, " NN : 0x%x\n", ControllerData
->Nn
));
446 Create IO completion queue.
448 @param[in] Private The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
450 @return EFI_SUCCESS Successfully create io completion queue.
451 @return others Fail to create io completion queue.
455 NvmeCreateIoCompletionQueue (
456 IN PEI_NVME_CONTROLLER_PRIVATE_DATA
*Private
459 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket
;
460 EFI_NVM_EXPRESS_COMMAND Command
;
461 EFI_NVM_EXPRESS_COMPLETION Completion
;
463 NVME_ADMIN_CRIOCQ CrIoCq
;
465 ZeroMem (&CommandPacket
, sizeof (EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET
));
466 ZeroMem (&Command
, sizeof (EFI_NVM_EXPRESS_COMMAND
));
467 ZeroMem (&Completion
, sizeof (EFI_NVM_EXPRESS_COMPLETION
));
468 ZeroMem (&CrIoCq
, sizeof (NVME_ADMIN_CRIOCQ
));
470 CommandPacket
.NvmeCmd
= &Command
;
471 CommandPacket
.NvmeCompletion
= &Completion
;
473 Command
.Cdw0
.Opcode
= NVME_ADMIN_CRIOCQ_CMD
;
474 CommandPacket
.TransferBuffer
= Private
->CqBuffer
[NVME_IO_QUEUE
];
475 CommandPacket
.TransferLength
= EFI_PAGE_SIZE
;
476 CommandPacket
.CommandTimeout
= NVME_GENERIC_TIMEOUT
;
477 CommandPacket
.QueueType
= NVME_ADMIN_QUEUE
;
479 CrIoCq
.Qid
= NVME_IO_QUEUE
;
480 CrIoCq
.Qsize
= NVME_CCQ_SIZE
;
482 CopyMem (&CommandPacket
.NvmeCmd
->Cdw10
, &CrIoCq
, sizeof (NVME_ADMIN_CRIOCQ
));
483 CommandPacket
.NvmeCmd
->Flags
= CDW10_VALID
| CDW11_VALID
;
485 Status
= NvmePassThruExecute (
487 NVME_CONTROLLER_NSID
,
494 Create IO submission queue.
496 @param[in] Private The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
498 @return EFI_SUCCESS Successfully create io submission queue.
499 @return others Fail to create io submission queue.
503 NvmeCreateIoSubmissionQueue (
504 IN PEI_NVME_CONTROLLER_PRIVATE_DATA
*Private
507 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket
;
508 EFI_NVM_EXPRESS_COMMAND Command
;
509 EFI_NVM_EXPRESS_COMPLETION Completion
;
511 NVME_ADMIN_CRIOSQ CrIoSq
;
513 ZeroMem (&CommandPacket
, sizeof (EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET
));
514 ZeroMem (&Command
, sizeof (EFI_NVM_EXPRESS_COMMAND
));
515 ZeroMem (&Completion
, sizeof (EFI_NVM_EXPRESS_COMPLETION
));
516 ZeroMem (&CrIoSq
, sizeof (NVME_ADMIN_CRIOSQ
));
518 CommandPacket
.NvmeCmd
= &Command
;
519 CommandPacket
.NvmeCompletion
= &Completion
;
521 Command
.Cdw0
.Opcode
= NVME_ADMIN_CRIOSQ_CMD
;
522 CommandPacket
.TransferBuffer
= Private
->SqBuffer
[NVME_IO_QUEUE
];
523 CommandPacket
.TransferLength
= EFI_PAGE_SIZE
;
524 CommandPacket
.CommandTimeout
= NVME_GENERIC_TIMEOUT
;
525 CommandPacket
.QueueType
= NVME_ADMIN_QUEUE
;
527 CrIoSq
.Qid
= NVME_IO_QUEUE
;
528 CrIoSq
.Qsize
= NVME_CSQ_SIZE
;
530 CrIoSq
.Cqid
= NVME_IO_QUEUE
;
532 CopyMem (&CommandPacket
.NvmeCmd
->Cdw10
, &CrIoSq
, sizeof (NVME_ADMIN_CRIOSQ
));
533 CommandPacket
.NvmeCmd
->Flags
= CDW10_VALID
| CDW11_VALID
;
535 Status
= NvmePassThruExecute (
537 NVME_CONTROLLER_NSID
,
544 Initialize the Nvm Express controller.
546 @param[in] Private The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
548 @retval EFI_SUCCESS The NVM Express Controller is initialized successfully.
549 @retval Others A device error occurred while initializing the controller.
554 IN PEI_NVME_CONTROLLER_PRIVATE_DATA
*Private
565 // Dump the NVME controller implementation version
567 NVME_GET_VER (Private
, &Ver
);
568 DEBUG ((DEBUG_INFO
, "NVME controller implementation version: %d.%d\n", Ver
.Mjr
, Ver
.Mnr
));
571 // Read the controller Capabilities register and verify that the NVM command set is supported
573 NVME_GET_CAP (Private
, &Private
->Cap
);
574 if ((Private
->Cap
.Css
& BIT0
) == 0) {
575 DEBUG ((DEBUG_ERROR
, "%a: The NVME controller doesn't support NVMe command set.\n", __FUNCTION__
));
576 return EFI_UNSUPPORTED
;
580 // Currently, the driver only supports 4k page size
582 if ((Private
->Cap
.Mpsmin
+ 12) > EFI_PAGE_SHIFT
) {
583 DEBUG ((DEBUG_ERROR
, "%a: The driver doesn't support page size other than 4K.\n", __FUNCTION__
));
585 return EFI_UNSUPPORTED
;
588 for (Index
= 0; Index
< NVME_MAX_QUEUES
; Index
++) {
589 Private
->Pt
[Index
] = 0;
590 Private
->Cid
[Index
] = 0;
591 ZeroMem ((VOID
*)(UINTN
)(&Private
->SqTdbl
[Index
]), sizeof (NVME_SQTDBL
));
592 ZeroMem ((VOID
*)(UINTN
)(&Private
->CqHdbl
[Index
]), sizeof (NVME_CQHDBL
));
595 ZeroMem (Private
->Buffer
, EFI_PAGE_SIZE
* NVME_MEM_MAX_PAGES
);
598 // Disable the NVME controller first
600 Status
= NvmeDisableController (Private
);
601 if (EFI_ERROR (Status
)) {
602 DEBUG ((DEBUG_ERROR
, "%a: NvmeDisableController fail, Status - %r\n", __FUNCTION__
, Status
));
607 // Set the number of entries in admin submission & completion queues
609 Aqa
.Asqs
= NVME_ASQ_SIZE
;
611 Aqa
.Acqs
= NVME_ACQ_SIZE
;
615 // Address of admin submission & completion queues
617 Asq
= (UINT64
)(UINTN
)(NVME_ASQ_BASE (Private
) & ~0xFFF);
618 Acq
= (UINT64
)(UINTN
)(NVME_ACQ_BASE (Private
) & ~0xFFF);
621 // Address of I/O submission & completion queues
623 Private
->SqBuffer
[0] = (NVME_SQ
*)(UINTN
)NVME_ASQ_BASE (Private
); // NVME_ADMIN_QUEUE
624 Private
->CqBuffer
[0] = (NVME_CQ
*)(UINTN
)NVME_ACQ_BASE (Private
); // NVME_ADMIN_QUEUE
625 Private
->SqBuffer
[1] = (NVME_SQ
*)(UINTN
)NVME_SQ_BASE (Private
, 0); // NVME_IO_QUEUE
626 Private
->CqBuffer
[1] = (NVME_CQ
*)(UINTN
)NVME_CQ_BASE (Private
, 0); // NVME_IO_QUEUE
627 DEBUG ((DEBUG_INFO
, "Admin Submission Queue Size (Aqa.Asqs) = [%08X]\n", Aqa
.Asqs
));
628 DEBUG ((DEBUG_INFO
, "Admin Completion Queue Size (Aqa.Acqs) = [%08X]\n", Aqa
.Acqs
));
629 DEBUG ((DEBUG_INFO
, "Admin Submission Queue (SqBuffer[0]) = [%08X]\n", Private
->SqBuffer
[0]));
630 DEBUG ((DEBUG_INFO
, "Admin Completion Queue (CqBuffer[0]) = [%08X]\n", Private
->CqBuffer
[0]));
631 DEBUG ((DEBUG_INFO
, "I/O Submission Queue (SqBuffer[1]) = [%08X]\n", Private
->SqBuffer
[1]));
632 DEBUG ((DEBUG_INFO
, "I/O Completion Queue (CqBuffer[1]) = [%08X]\n", Private
->CqBuffer
[1]));
635 // Program admin queue attributes
637 NVME_SET_AQA (Private
, &Aqa
);
640 // Program admin submission & completion queues address
642 NVME_SET_ASQ (Private
, &Asq
);
643 NVME_SET_ACQ (Private
, &Acq
);
646 // Enable the NVME controller
648 Status
= NvmeEnableController (Private
);
649 if (EFI_ERROR (Status
)) {
650 DEBUG ((DEBUG_ERROR
, "%a: NvmeEnableController fail, Status - %r\n", __FUNCTION__
, Status
));
655 // Get the Identify Controller data
657 if (Private
->ControllerData
== NULL
) {
658 Private
->ControllerData
= (NVME_ADMIN_CONTROLLER_DATA
*)AllocateZeroPool (sizeof (NVME_ADMIN_CONTROLLER_DATA
));
659 if (Private
->ControllerData
== NULL
) {
660 return EFI_OUT_OF_RESOURCES
;
664 Status
= NvmeIdentifyController (Private
, Private
->ControllerData
);
665 if (EFI_ERROR (Status
)) {
666 DEBUG ((DEBUG_ERROR
, "%a: NvmeIdentifyController fail, Status - %r\n", __FUNCTION__
, Status
));
670 NvmeDumpControllerData (Private
->ControllerData
);
673 // Check the namespace number for storing the namespaces information
675 if (Private
->ControllerData
->Nn
> MAX_UINT32
/ sizeof (PEI_NVME_NAMESPACE_INFO
)) {
678 "%a: Number of Namespaces field in Identify Controller data not supported by the driver.\n",
681 return EFI_UNSUPPORTED
;
685 // Create one I/O completion queue and one I/O submission queue
687 Status
= NvmeCreateIoCompletionQueue (Private
);
688 if (EFI_ERROR (Status
)) {
689 DEBUG ((DEBUG_ERROR
, "%a: Create IO completion queue fail, Status - %r\n", __FUNCTION__
, Status
));
693 Status
= NvmeCreateIoSubmissionQueue (Private
);
694 if (EFI_ERROR (Status
)) {
695 DEBUG ((DEBUG_ERROR
, "%a: Create IO submission queue fail, Status - %r\n", __FUNCTION__
, Status
));
702 Free the DMA resources allocated by an NVME controller.
704 @param[in] Private The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
708 NvmeFreeDmaResource (
709 IN PEI_NVME_CONTROLLER_PRIVATE_DATA
*Private
712 ASSERT (Private
!= NULL
);
714 if (Private
->BufferMapping
!= NULL
) {
718 Private
->BufferMapping