2 The NvmExpressPei driver is used to manage non-volatile memory subsystem
3 which follows NVM Express specification at PEI phase.
5 Copyright (c) 2018 - 2019, Intel Corporation. All rights reserved.<BR>
7 SPDX-License-Identifier: BSD-2-Clause-Patent
11 #include "NvmExpressPei.h"
14 Transfer MMIO Data to memory.
16 @param[in,out] MemBuffer Destination: Memory address.
17 @param[in] MmioAddr Source: MMIO address.
18 @param[in] Size Size for read.
20 @retval EFI_SUCCESS MMIO read sucessfully.
25 IN OUT VOID
*MemBuffer
,
34 // priority has adjusted
37 *((UINT32
*)MemBuffer
) = MmioRead32 (MmioAddr
);
41 *((UINT64
*)MemBuffer
) = MmioRead64 (MmioAddr
);
45 *((UINT16
*)MemBuffer
) = MmioRead16 (MmioAddr
);
49 *((UINT8
*)MemBuffer
) = MmioRead8 (MmioAddr
);
53 Ptr
= (UINT8
*)MemBuffer
;
54 for (Offset
= 0; Offset
< Size
; Offset
+= 1) {
55 Data
= MmioRead8 (MmioAddr
+ Offset
);
65 Transfer memory data to MMIO.
67 @param[in,out] MmioAddr Destination: MMIO address.
68 @param[in] MemBuffer Source: Memory address.
69 @param[in] Size Size for write.
71 @retval EFI_SUCCESS MMIO write sucessfully.
76 IN OUT UINTN MmioAddr
,
85 // priority has adjusted
88 MmioWrite32 (MmioAddr
, *((UINT32
*)MemBuffer
));
92 MmioWrite64 (MmioAddr
, *((UINT64
*)MemBuffer
));
96 MmioWrite16 (MmioAddr
, *((UINT16
*)MemBuffer
));
100 MmioWrite8 (MmioAddr
, *((UINT8
*)MemBuffer
));
104 Ptr
= (UINT8
*)MemBuffer
;
105 for (Offset
= 0; Offset
< Size
; Offset
+= 1) {
107 MmioWrite8 (MmioAddr
+ Offset
, Data
);
116 Get the page offset for specific NVME based memory.
118 @param[in] BaseMemIndex The Index of BaseMem (0-based).
120 @retval - The page count for specific BaseMem Index
124 NvmeBaseMemPageOffset (
125 IN UINTN BaseMemIndex
130 UINT32 PageSizeList
[5];
132 PageSizeList
[0] = 1; /* ASQ */
133 PageSizeList
[1] = 1; /* ACQ */
134 PageSizeList
[2] = 1; /* SQs */
135 PageSizeList
[3] = 1; /* CQs */
136 PageSizeList
[4] = NVME_PRP_SIZE
; /* PRPs */
138 if (BaseMemIndex
> MAX_BASEMEM_COUNT
) {
139 DEBUG ((DEBUG_ERROR
, "%a: The input BaseMem index is invalid.\n", __FUNCTION__
));
145 for (Index
= 0; Index
< BaseMemIndex
; Index
++) {
146 Pages
+= PageSizeList
[Index
];
153 Wait for NVME controller status to be ready or not.
155 @param[in] Private The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
156 @param[in] WaitReady Flag for waitting status ready or not.
158 @return EFI_SUCCESS Successfully to wait specific status.
159 @return others Fail to wait for specific controller status.
164 IN PEI_NVME_CONTROLLER_PRIVATE_DATA
*Private
,
174 // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to set after
175 // Cc.Enable. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To.
177 if (Private
->Cap
.To
== 0) {
180 Timeout
= Private
->Cap
.To
;
183 Status
= EFI_SUCCESS
;
184 for(Index
= (Timeout
* 500); Index
!= 0; --Index
) {
185 MicroSecondDelay (1000);
188 // Check if the controller is initialized
190 Status
= NVME_GET_CSTS (Private
, &Csts
);
191 if (EFI_ERROR(Status
)) {
192 DEBUG ((DEBUG_ERROR
, "%a: NVME_GET_CSTS fail, Status - %r\n", __FUNCTION__
, Status
));
196 if ((BOOLEAN
) Csts
.Rdy
== WaitReady
) {
202 Status
= EFI_TIMEOUT
;
209 Disable the Nvm Express controller.
211 @param[in] Private The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
213 @return EFI_SUCCESS Successfully disable the controller.
214 @return others Fail to disable the controller.
218 NvmeDisableController (
219 IN PEI_NVME_CONTROLLER_PRIVATE_DATA
*Private
226 Status
= NVME_GET_CSTS (Private
, &Csts
);
229 // Read Controller Configuration Register.
231 Status
= NVME_GET_CC (Private
, &Cc
);
232 if (EFI_ERROR (Status
)) {
233 DEBUG ((DEBUG_ERROR
, "%a: NVME_GET_CC fail, Status - %r\n", __FUNCTION__
, Status
));
240 // Disable the controller.
242 Status
= NVME_SET_CC (Private
, &Cc
);
243 if (EFI_ERROR (Status
)) {
244 DEBUG ((DEBUG_ERROR
, "%a: NVME_SET_CC fail, Status - %r\n", __FUNCTION__
, Status
));
249 Status
= NvmeWaitController (Private
, FALSE
);
250 if (EFI_ERROR (Status
)) {
251 DEBUG ((DEBUG_ERROR
, "%a: NvmeWaitController fail, Status - %r\n", __FUNCTION__
, Status
));
258 DEBUG ((DEBUG_ERROR
, "%a fail, Status - %r\n", __FUNCTION__
, Status
));
263 Enable the Nvm Express controller.
265 @param[in] Private The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
267 @return EFI_SUCCESS Successfully enable the controller.
268 @return EFI_DEVICE_ERROR Fail to enable the controller.
269 @return EFI_TIMEOUT Fail to enable the controller in given time slot.
273 NvmeEnableController (
274 IN PEI_NVME_CONTROLLER_PRIVATE_DATA
*Private
281 // Enable the controller
282 // CC.AMS, CC.MPS and CC.CSS are all set to 0
284 ZeroMem (&Cc
, sizeof (NVME_CC
));
288 Status
= NVME_SET_CC (Private
, &Cc
);
289 if (EFI_ERROR (Status
)) {
290 DEBUG ((DEBUG_ERROR
, "%a: NVME_SET_CC fail, Status - %r\n", __FUNCTION__
, Status
));
294 Status
= NvmeWaitController (Private
, TRUE
);
295 if (EFI_ERROR (Status
)) {
296 DEBUG ((DEBUG_ERROR
, "%a: NvmeWaitController fail, Status - %r\n", __FUNCTION__
, Status
));
303 DEBUG ((DEBUG_ERROR
, "%a fail, Status: %r\n", __FUNCTION__
, Status
));
308 Get the Identify Controller data.
310 @param[in] Private The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
311 @param[in] Buffer The Buffer used to store the Identify Controller data.
313 @return EFI_SUCCESS Successfully get the Identify Controller data.
314 @return others Fail to get the Identify Controller data.
318 NvmeIdentifyController (
319 IN PEI_NVME_CONTROLLER_PRIVATE_DATA
*Private
,
323 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket
;
324 EFI_NVM_EXPRESS_COMMAND Command
;
325 EFI_NVM_EXPRESS_COMPLETION Completion
;
328 ZeroMem (&CommandPacket
, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET
));
329 ZeroMem (&Command
, sizeof(EFI_NVM_EXPRESS_COMMAND
));
330 ZeroMem (&Completion
, sizeof(EFI_NVM_EXPRESS_COMPLETION
));
332 Command
.Cdw0
.Opcode
= NVME_ADMIN_IDENTIFY_CMD
;
334 // According to Nvm Express 1.1 spec Figure 38, When not used, the field shall be cleared to 0h.
335 // For the Identify command, the Namespace Identifier is only used for the Namespace Data structure.
339 CommandPacket
.NvmeCmd
= &Command
;
340 CommandPacket
.NvmeCompletion
= &Completion
;
341 CommandPacket
.TransferBuffer
= Buffer
;
342 CommandPacket
.TransferLength
= sizeof (NVME_ADMIN_CONTROLLER_DATA
);
343 CommandPacket
.CommandTimeout
= NVME_GENERIC_TIMEOUT
;
344 CommandPacket
.QueueType
= NVME_ADMIN_QUEUE
;
346 // Set bit 0 (Cns bit) to 1 to identify the controller
348 CommandPacket
.NvmeCmd
->Cdw10
= 1;
349 CommandPacket
.NvmeCmd
->Flags
= CDW10_VALID
;
351 Status
= NvmePassThruExecute (
353 NVME_CONTROLLER_NSID
,
360 Get specified identify namespace data.
362 @param[in] Private The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
363 @param[in] NamespaceId The specified namespace identifier.
364 @param[in] Buffer The buffer used to store the identify namespace data.
366 @return EFI_SUCCESS Successfully get the identify namespace data.
367 @return EFI_DEVICE_ERROR Fail to get the identify namespace data.
371 NvmeIdentifyNamespace (
372 IN PEI_NVME_CONTROLLER_PRIVATE_DATA
*Private
,
373 IN UINT32 NamespaceId
,
377 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket
;
378 EFI_NVM_EXPRESS_COMMAND Command
;
379 EFI_NVM_EXPRESS_COMPLETION Completion
;
382 ZeroMem (&CommandPacket
, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET
));
383 ZeroMem (&Command
, sizeof(EFI_NVM_EXPRESS_COMMAND
));
384 ZeroMem (&Completion
, sizeof(EFI_NVM_EXPRESS_COMPLETION
));
386 Command
.Cdw0
.Opcode
= NVME_ADMIN_IDENTIFY_CMD
;
387 Command
.Nsid
= NamespaceId
;
389 CommandPacket
.NvmeCmd
= &Command
;
390 CommandPacket
.NvmeCompletion
= &Completion
;
391 CommandPacket
.TransferBuffer
= Buffer
;
392 CommandPacket
.TransferLength
= sizeof (NVME_ADMIN_NAMESPACE_DATA
);
393 CommandPacket
.CommandTimeout
= NVME_GENERIC_TIMEOUT
;
394 CommandPacket
.QueueType
= NVME_ADMIN_QUEUE
;
396 // Set bit 0 (Cns bit) to 1 to identify a namespace
398 CommandPacket
.NvmeCmd
->Cdw10
= 0;
399 CommandPacket
.NvmeCmd
->Flags
= CDW10_VALID
;
401 Status
= NvmePassThruExecute (
410 Dump the Identify Controller data.
412 @param[in] ControllerData The pointer to the NVME_ADMIN_CONTROLLER_DATA data structure.
416 NvmeDumpControllerData (
417 IN NVME_ADMIN_CONTROLLER_DATA
*ControllerData
423 CopyMem (Sn
, ControllerData
->Sn
, sizeof (ControllerData
->Sn
));
425 CopyMem (Mn
, ControllerData
->Mn
, sizeof (ControllerData
->Mn
));
428 DEBUG ((DEBUG_INFO
, " == NVME IDENTIFY CONTROLLER DATA ==\n"));
429 DEBUG ((DEBUG_INFO
, " PCI VID : 0x%x\n", ControllerData
->Vid
));
430 DEBUG ((DEBUG_INFO
, " PCI SSVID : 0x%x\n", ControllerData
->Ssvid
));
431 DEBUG ((DEBUG_INFO
, " SN : %a\n", Sn
));
432 DEBUG ((DEBUG_INFO
, " MN : %a\n", Mn
));
433 DEBUG ((DEBUG_INFO
, " FR : 0x%lx\n", *((UINT64
*)ControllerData
->Fr
)));
434 DEBUG ((DEBUG_INFO
, " RAB : 0x%x\n", ControllerData
->Rab
));
435 DEBUG ((DEBUG_INFO
, " IEEE : 0x%x\n", *(UINT32
*)ControllerData
->Ieee_oui
));
436 DEBUG ((DEBUG_INFO
, " AERL : 0x%x\n", ControllerData
->Aerl
));
437 DEBUG ((DEBUG_INFO
, " SQES : 0x%x\n", ControllerData
->Sqes
));
438 DEBUG ((DEBUG_INFO
, " CQES : 0x%x\n", ControllerData
->Cqes
));
439 DEBUG ((DEBUG_INFO
, " NN : 0x%x\n", ControllerData
->Nn
));
444 Create IO completion queue.
446 @param[in] Private The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
448 @return EFI_SUCCESS Successfully create io completion queue.
449 @return others Fail to create io completion queue.
453 NvmeCreateIoCompletionQueue (
454 IN PEI_NVME_CONTROLLER_PRIVATE_DATA
*Private
457 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket
;
458 EFI_NVM_EXPRESS_COMMAND Command
;
459 EFI_NVM_EXPRESS_COMPLETION Completion
;
461 NVME_ADMIN_CRIOCQ CrIoCq
;
463 ZeroMem (&CommandPacket
, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET
));
464 ZeroMem (&Command
, sizeof(EFI_NVM_EXPRESS_COMMAND
));
465 ZeroMem (&Completion
, sizeof(EFI_NVM_EXPRESS_COMPLETION
));
466 ZeroMem (&CrIoCq
, sizeof(NVME_ADMIN_CRIOCQ
));
468 CommandPacket
.NvmeCmd
= &Command
;
469 CommandPacket
.NvmeCompletion
= &Completion
;
471 Command
.Cdw0
.Opcode
= NVME_ADMIN_CRIOCQ_CMD
;
472 CommandPacket
.TransferBuffer
= Private
->CqBuffer
[NVME_IO_QUEUE
];
473 CommandPacket
.TransferLength
= EFI_PAGE_SIZE
;
474 CommandPacket
.CommandTimeout
= NVME_GENERIC_TIMEOUT
;
475 CommandPacket
.QueueType
= NVME_ADMIN_QUEUE
;
477 CrIoCq
.Qid
= NVME_IO_QUEUE
;
478 CrIoCq
.Qsize
= NVME_CCQ_SIZE
;
480 CopyMem (&CommandPacket
.NvmeCmd
->Cdw10
, &CrIoCq
, sizeof (NVME_ADMIN_CRIOCQ
));
481 CommandPacket
.NvmeCmd
->Flags
= CDW10_VALID
| CDW11_VALID
;
483 Status
= NvmePassThruExecute (
485 NVME_CONTROLLER_NSID
,
492 Create IO submission queue.
494 @param[in] Private The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
496 @return EFI_SUCCESS Successfully create io submission queue.
497 @return others Fail to create io submission queue.
501 NvmeCreateIoSubmissionQueue (
502 IN PEI_NVME_CONTROLLER_PRIVATE_DATA
*Private
505 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket
;
506 EFI_NVM_EXPRESS_COMMAND Command
;
507 EFI_NVM_EXPRESS_COMPLETION Completion
;
509 NVME_ADMIN_CRIOSQ CrIoSq
;
511 ZeroMem (&CommandPacket
, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET
));
512 ZeroMem (&Command
, sizeof(EFI_NVM_EXPRESS_COMMAND
));
513 ZeroMem (&Completion
, sizeof(EFI_NVM_EXPRESS_COMPLETION
));
514 ZeroMem (&CrIoSq
, sizeof(NVME_ADMIN_CRIOSQ
));
516 CommandPacket
.NvmeCmd
= &Command
;
517 CommandPacket
.NvmeCompletion
= &Completion
;
519 Command
.Cdw0
.Opcode
= NVME_ADMIN_CRIOSQ_CMD
;
520 CommandPacket
.TransferBuffer
= Private
->SqBuffer
[NVME_IO_QUEUE
];
521 CommandPacket
.TransferLength
= EFI_PAGE_SIZE
;
522 CommandPacket
.CommandTimeout
= NVME_GENERIC_TIMEOUT
;
523 CommandPacket
.QueueType
= NVME_ADMIN_QUEUE
;
525 CrIoSq
.Qid
= NVME_IO_QUEUE
;
526 CrIoSq
.Qsize
= NVME_CSQ_SIZE
;
528 CrIoSq
.Cqid
= NVME_IO_QUEUE
;
530 CopyMem (&CommandPacket
.NvmeCmd
->Cdw10
, &CrIoSq
, sizeof (NVME_ADMIN_CRIOSQ
));
531 CommandPacket
.NvmeCmd
->Flags
= CDW10_VALID
| CDW11_VALID
;
533 Status
= NvmePassThruExecute (
535 NVME_CONTROLLER_NSID
,
542 Initialize the Nvm Express controller.
544 @param[in] Private The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
546 @retval EFI_SUCCESS The NVM Express Controller is initialized successfully.
547 @retval Others A device error occurred while initializing the controller.
552 IN PEI_NVME_CONTROLLER_PRIVATE_DATA
*Private
563 // Dump the NVME controller implementation version
565 NVME_GET_VER (Private
, &Ver
);
566 DEBUG ((DEBUG_INFO
, "NVME controller implementation version: %d.%d\n", Ver
.Mjr
, Ver
.Mnr
));
569 // Read the controller Capabilities register and verify that the NVM command set is supported
571 NVME_GET_CAP (Private
, &Private
->Cap
);
572 if (Private
->Cap
.Css
!= 0x01) {
573 DEBUG ((DEBUG_ERROR
, "%a: The NVME controller doesn't support NVMe command set.\n", __FUNCTION__
));
574 return EFI_UNSUPPORTED
;
578 // Currently, the driver only supports 4k page size
580 if ((Private
->Cap
.Mpsmin
+ 12) > EFI_PAGE_SHIFT
) {
581 DEBUG ((DEBUG_ERROR
, "%a: The driver doesn't support page size other than 4K.\n", __FUNCTION__
));
583 return EFI_UNSUPPORTED
;
586 for (Index
= 0; Index
< NVME_MAX_QUEUES
; Index
++) {
587 Private
->Pt
[Index
] = 0;
588 Private
->Cid
[Index
] = 0;
589 ZeroMem ((VOID
*)(UINTN
)(&Private
->SqTdbl
[Index
]), sizeof (NVME_SQTDBL
));
590 ZeroMem ((VOID
*)(UINTN
)(&Private
->CqHdbl
[Index
]), sizeof (NVME_CQHDBL
));
592 ZeroMem (Private
->Buffer
, EFI_PAGE_SIZE
* NVME_MEM_MAX_PAGES
);
595 // Disable the NVME controller first
597 Status
= NvmeDisableController (Private
);
598 if (EFI_ERROR (Status
)) {
599 DEBUG ((DEBUG_ERROR
, "%a: NvmeDisableController fail, Status - %r\n", __FUNCTION__
, Status
));
604 // Set the number of entries in admin submission & completion queues
606 Aqa
.Asqs
= NVME_ASQ_SIZE
;
608 Aqa
.Acqs
= NVME_ACQ_SIZE
;
612 // Address of admin submission & completion queues
614 Asq
= (UINT64
)(UINTN
)(NVME_ASQ_BASE (Private
) & ~0xFFF);
615 Acq
= (UINT64
)(UINTN
)(NVME_ACQ_BASE (Private
) & ~0xFFF);
618 // Address of I/O submission & completion queues
620 Private
->SqBuffer
[0] = (NVME_SQ
*)(UINTN
)NVME_ASQ_BASE (Private
); // NVME_ADMIN_QUEUE
621 Private
->CqBuffer
[0] = (NVME_CQ
*)(UINTN
)NVME_ACQ_BASE (Private
); // NVME_ADMIN_QUEUE
622 Private
->SqBuffer
[1] = (NVME_SQ
*)(UINTN
)NVME_SQ_BASE (Private
, 0); // NVME_IO_QUEUE
623 Private
->CqBuffer
[1] = (NVME_CQ
*)(UINTN
)NVME_CQ_BASE (Private
, 0); // NVME_IO_QUEUE
624 DEBUG ((DEBUG_INFO
, "Admin Submission Queue Size (Aqa.Asqs) = [%08X]\n", Aqa
.Asqs
));
625 DEBUG ((DEBUG_INFO
, "Admin Completion Queue Size (Aqa.Acqs) = [%08X]\n", Aqa
.Acqs
));
626 DEBUG ((DEBUG_INFO
, "Admin Submission Queue (SqBuffer[0]) = [%08X]\n", Private
->SqBuffer
[0]));
627 DEBUG ((DEBUG_INFO
, "Admin Completion Queue (CqBuffer[0]) = [%08X]\n", Private
->CqBuffer
[0]));
628 DEBUG ((DEBUG_INFO
, "I/O Submission Queue (SqBuffer[1]) = [%08X]\n", Private
->SqBuffer
[1]));
629 DEBUG ((DEBUG_INFO
, "I/O Completion Queue (CqBuffer[1]) = [%08X]\n", Private
->CqBuffer
[1]));
632 // Program admin queue attributes
634 NVME_SET_AQA (Private
, &Aqa
);
637 // Program admin submission & completion queues address
639 NVME_SET_ASQ (Private
, &Asq
);
640 NVME_SET_ACQ (Private
, &Acq
);
643 // Enable the NVME controller
645 Status
= NvmeEnableController (Private
);
646 if (EFI_ERROR (Status
)) {
647 DEBUG ((DEBUG_ERROR
, "%a: NvmeEnableController fail, Status - %r\n", __FUNCTION__
, Status
));
652 // Get the Identify Controller data
654 if (Private
->ControllerData
== NULL
) {
655 Private
->ControllerData
= (NVME_ADMIN_CONTROLLER_DATA
*)AllocateZeroPool (sizeof (NVME_ADMIN_CONTROLLER_DATA
));
656 if (Private
->ControllerData
== NULL
) {
657 return EFI_OUT_OF_RESOURCES
;
660 Status
= NvmeIdentifyController (Private
, Private
->ControllerData
);
661 if (EFI_ERROR (Status
)) {
662 DEBUG ((DEBUG_ERROR
, "%a: NvmeIdentifyController fail, Status - %r\n", __FUNCTION__
, Status
));
665 NvmeDumpControllerData (Private
->ControllerData
);
668 // Check the namespace number for storing the namespaces information
670 if (Private
->ControllerData
->Nn
> MAX_UINT32
/ sizeof (PEI_NVME_NAMESPACE_INFO
)) {
673 "%a: Number of Namespaces field in Identify Controller data not supported by the driver.\n",
676 return EFI_UNSUPPORTED
;
680 // Create one I/O completion queue and one I/O submission queue
682 Status
= NvmeCreateIoCompletionQueue (Private
);
683 if (EFI_ERROR (Status
)) {
684 DEBUG ((DEBUG_ERROR
, "%a: Create IO completion queue fail, Status - %r\n", __FUNCTION__
, Status
));
687 Status
= NvmeCreateIoSubmissionQueue (Private
);
688 if (EFI_ERROR (Status
)) {
689 DEBUG ((DEBUG_ERROR
, "%a: Create IO submission queue fail, Status - %r\n", __FUNCTION__
, Status
));
696 Free the DMA resources allocated by an NVME controller.
698 @param[in] Private The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
702 NvmeFreeDmaResource (
703 IN PEI_NVME_CONTROLLER_PRIVATE_DATA
*Private
706 ASSERT (Private
!= NULL
);
708 if (Private
->BufferMapping
!= NULL
) {
712 Private
->BufferMapping