[mirror_edk2.git] / MdeModulePkg / Bus / Pci / NvmExpressPei / NvmExpressPeiHci.c

/** @file\r
  The NvmExpressPei driver is used to manage non-volatile memory subsystem\r
  which follows NVM Express specification at PEI phase.\r
\r
  Copyright (c) 2018 - 2019, Intel Corporation. All rights reserved.<BR>\r
\r
  SPDX-License-Identifier: BSD-2-Clause-Patent\r
\r
**/\r
\r
#include "NvmExpressPei.h"\r
\r
/**\r
  Transfer MMIO Data to memory.\r
\r
  @param[in,out] MemBuffer    Destination: Memory address.\r
  @param[in] MmioAddr         Source: MMIO address.\r
  @param[in] Size             Size for read.\r
\r
  @retval EFI_SUCCESS         MMIO read sucessfully.\r
\r
**/\r
EFI_STATUS\r
NvmeMmioRead (\r
  IN OUT VOID   *MemBuffer,\r
  IN     UINTN  MmioAddr,\r
  IN     UINTN  Size\r
  )\r
{\r
  UINTN  Offset;\r
  UINT8  Data;\r
  UINT8  *Ptr;\r
\r
  // priority has adjusted\r
  switch (Size) {\r
    case 4:\r
      *((UINT32 *)MemBuffer) = MmioRead32 (MmioAddr);\r
      break;\r
\r
    case 8:\r
      *((UINT64 *)MemBuffer) = MmioRead64 (MmioAddr);\r
      break;\r
\r
    case 2:\r
      *((UINT16 *)MemBuffer) = MmioRead16 (MmioAddr);\r
      break;\r
\r
    case 1:\r
      *((UINT8 *)MemBuffer) = MmioRead8 (MmioAddr);\r
      break;\r
\r
    default:\r
      Ptr = (UINT8 *)MemBuffer;\r
      for (Offset = 0; Offset < Size; Offset += 1) {\r
        Data        = MmioRead8 (MmioAddr + Offset);\r
        Ptr[Offset] = Data;\r
      }\r
\r
      break;\r
  }\r
\r
  return EFI_SUCCESS;\r
}\r
\r
/**\r
  Transfer memory data to MMIO.\r
\r
  @param[in,out] MmioAddr    Destination: MMIO address.\r
  @param[in] MemBuffer       Source: Memory address.\r
  @param[in] Size            Size for write.\r
\r
  @retval EFI_SUCCESS        MMIO write sucessfully.\r
\r
**/\r
EFI_STATUS\r
NvmeMmioWrite (\r
  IN OUT UINTN  MmioAddr,\r
  IN     VOID   *MemBuffer,\r
  IN     UINTN  Size\r
  )\r
{\r
  UINTN  Offset;\r
  UINT8  Data;\r
  UINT8  *Ptr;\r
\r
  // priority has adjusted\r
  switch (Size) {\r
    case 4:\r
      MmioWrite32 (MmioAddr, *((UINT32 *)MemBuffer));\r
      break;\r
\r
    case 8:\r
      MmioWrite64 (MmioAddr, *((UINT64 *)MemBuffer));\r
      break;\r
\r
    case 2:\r
      MmioWrite16 (MmioAddr, *((UINT16 *)MemBuffer));\r
      break;\r
\r
    case 1:\r
      MmioWrite8 (MmioAddr, *((UINT8 *)MemBuffer));\r
      break;\r
\r
    default:\r
      Ptr = (UINT8 *)MemBuffer;\r
      for (Offset = 0; Offset < Size; Offset += 1) {\r
        Data = Ptr[Offset];\r
        MmioWrite8 (MmioAddr + Offset, Data);\r
      }\r
\r
      break;\r
  }\r
\r
  return EFI_SUCCESS;\r
}\r
\r
/**\r
  Get the page offset for specific NVME based memory.\r
\r
  @param[in] BaseMemIndex    The Index of BaseMem (0-based).\r
\r
  @retval - The page count for specific BaseMem Index\r
\r
**/\r
UINT32\r
NvmeBaseMemPageOffset (\r
  IN UINTN  BaseMemIndex\r
  )\r
{\r
  UINT32  Pages;\r
  UINTN   Index;\r
  UINT32  PageSizeList[5];\r
\r
  PageSizeList[0] = 1;             /* ASQ */\r
  PageSizeList[1] = 1;             /* ACQ */\r
  PageSizeList[2] = 1;             /* SQs */\r
  PageSizeList[3] = 1;             /* CQs */\r
  PageSizeList[4] = NVME_PRP_SIZE; /* PRPs */\r
\r
  if (BaseMemIndex > MAX_BASEMEM_COUNT) {\r
    DEBUG ((DEBUG_ERROR, "%a: The input BaseMem index is invalid.\n", __FUNCTION__));\r
    ASSERT (FALSE);\r
    return 0;\r
  }\r
\r
  Pages = 0;\r
  for (Index = 0; Index < BaseMemIndex; Index++) {\r
    Pages += PageSizeList[Index];\r
  }\r
\r
  return Pages;\r
}\r
\r
/**\r
  Wait for NVME controller status to be ready or not.\r
\r
  @param[in] Private      The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.\r
  @param[in] WaitReady    Flag for waitting status ready or not.\r
\r
  @return EFI_SUCCESS     Successfully to wait specific status.\r
  @return others          Fail to wait for specific controller status.\r
\r
**/\r
EFI_STATUS\r
NvmeWaitController (\r
  IN PEI_NVME_CONTROLLER_PRIVATE_DATA  *Private,\r
  IN BOOLEAN                           WaitReady\r
  )\r
{\r
  NVME_CSTS   Csts;\r
  EFI_STATUS  Status;\r
  UINT32      Index;\r
  UINT8       Timeout;\r
\r
  //\r
  // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to set after\r
  // Cc.Enable. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To.\r
  //\r
  if (Private->Cap.To == 0) {\r
    Timeout = 1;\r
  } else {\r
    Timeout = Private->Cap.To;\r
  }\r
\r
  Status = EFI_SUCCESS;\r
  for (Index = (Timeout * 500); Index != 0; --Index) {\r
    MicroSecondDelay (1000);\r
\r
    //\r
    // Check if the controller is initialized\r
    //\r
    Status = NVME_GET_CSTS (Private, &Csts);\r
    if (EFI_ERROR (Status)) {\r
      DEBUG ((DEBUG_ERROR, "%a: NVME_GET_CSTS fail, Status - %r\n", __FUNCTION__, Status));\r
      return Status;\r
    }\r
\r
    if ((BOOLEAN)Csts.Rdy == WaitReady) {\r
      break;\r
    }\r
  }\r
\r
  if (Index == 0) {\r
    Status = EFI_TIMEOUT;\r
  }\r
\r
  return Status;\r
}\r
\r
/**\r
  Disable the Nvm Express controller.\r
\r
  @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.\r
\r
  @return EFI_SUCCESS    Successfully disable the controller.\r
  @return others         Fail to disable the controller.\r
\r
**/\r
EFI_STATUS\r
NvmeDisableController (\r
  IN PEI_NVME_CONTROLLER_PRIVATE_DATA  *Private\r
  )\r
{\r
  NVME_CC     Cc;\r
  NVME_CSTS   Csts;\r
  EFI_STATUS  Status;\r
\r
  Status = NVME_GET_CSTS (Private, &Csts);\r
\r
  //\r
  // Read Controller Configuration Register.\r
  //\r
  Status = NVME_GET_CC (Private, &Cc);\r
  if (EFI_ERROR (Status)) {\r
    DEBUG ((DEBUG_ERROR, "%a: NVME_GET_CC fail, Status - %r\n", __FUNCTION__, Status));\r
    goto ErrorExit;\r
  }\r
\r
  if (Cc.En == 1) {\r
    Cc.En = 0;\r
    //\r
    // Disable the controller.\r
    //\r
    Status = NVME_SET_CC (Private, &Cc);\r
    if (EFI_ERROR (Status)) {\r
      DEBUG ((DEBUG_ERROR, "%a: NVME_SET_CC fail, Status - %r\n", __FUNCTION__, Status));\r
      goto ErrorExit;\r
    }\r
  }\r
\r
  Status = NvmeWaitController (Private, FALSE);\r
  if (EFI_ERROR (Status)) {\r
    DEBUG ((DEBUG_ERROR, "%a: NvmeWaitController fail, Status - %r\n", __FUNCTION__, Status));\r
    goto ErrorExit;\r
  }\r
\r
  return EFI_SUCCESS;\r
\r
ErrorExit:\r
  DEBUG ((DEBUG_ERROR, "%a fail, Status - %r\n", __FUNCTION__, Status));\r
  return Status;\r
}\r
\r
/**\r
  Enable the Nvm Express controller.\r
\r
  @param[in] Private    The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.\r
\r
  @return EFI_SUCCESS         Successfully enable the controller.\r
  @return EFI_DEVICE_ERROR    Fail to enable the controller.\r
  @return EFI_TIMEOUT         Fail to enable the controller in given time slot.\r
\r
**/\r
EFI_STATUS\r
NvmeEnableController (\r
  IN PEI_NVME_CONTROLLER_PRIVATE_DATA  *Private\r
  )\r
{\r
  NVME_CC     Cc;\r
  EFI_STATUS  Status;\r
\r
  //\r
  // Enable the controller\r
  // CC.AMS, CC.MPS and CC.CSS are all set to 0\r
  //\r
  ZeroMem (&Cc, sizeof (NVME_CC));\r
  Cc.En     = 1;\r
  Cc.Iosqes = 6;\r
  Cc.Iocqes = 4;\r
  Status    = NVME_SET_CC (Private, &Cc);\r
  if (EFI_ERROR (Status)) {\r
    DEBUG ((DEBUG_ERROR, "%a: NVME_SET_CC fail, Status - %r\n", __FUNCTION__, Status));\r
    goto ErrorExit;\r
  }\r
\r
  Status = NvmeWaitController (Private, TRUE);\r
  if (EFI_ERROR (Status)) {\r
    DEBUG ((DEBUG_ERROR, "%a: NvmeWaitController fail, Status - %r\n", __FUNCTION__, Status));\r
    goto ErrorExit;\r
  }\r
\r
  return EFI_SUCCESS;\r
\r
ErrorExit:\r
  DEBUG ((DEBUG_ERROR, "%a fail, Status: %r\n", __FUNCTION__, Status));\r
  return Status;\r
}\r
\r
/**\r
  Get the Identify Controller data.\r
\r
  @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.\r
  @param[in] Buffer      The Buffer used to store the Identify Controller data.\r
\r
  @return EFI_SUCCESS    Successfully get the Identify Controller data.\r
  @return others         Fail to get the Identify Controller data.\r
\r
**/\r
EFI_STATUS\r
NvmeIdentifyController (\r
  IN PEI_NVME_CONTROLLER_PRIVATE_DATA  *Private,\r
  IN VOID                              *Buffer\r
  )\r
{\r
  EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET  CommandPacket;\r
  EFI_NVM_EXPRESS_COMMAND                   Command;\r
  EFI_NVM_EXPRESS_COMPLETION                Completion;\r
  EFI_STATUS                                Status;\r
\r
  ZeroMem (&CommandPacket, sizeof (EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));\r
  ZeroMem (&Command, sizeof (EFI_NVM_EXPRESS_COMMAND));\r
  ZeroMem (&Completion, sizeof (EFI_NVM_EXPRESS_COMPLETION));\r
\r
  Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD;\r
  //\r
  // According to Nvm Express 1.1 spec Figure 38, When not used, the field shall be cleared to 0h.\r
  // For the Identify command, the Namespace Identifier is only used for the Namespace Data structure.\r
  //\r
  Command.Nsid = 0;\r
\r
  CommandPacket.NvmeCmd        = &Command;\r
  CommandPacket.NvmeCompletion = &Completion;\r
  CommandPacket.TransferBuffer = Buffer;\r
  CommandPacket.TransferLength = sizeof (NVME_ADMIN_CONTROLLER_DATA);\r
  CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;\r
  CommandPacket.QueueType      = NVME_ADMIN_QUEUE;\r
  //\r
  // Set bit 0 (Cns bit) to 1 to identify the controller\r
  //\r
  CommandPacket.NvmeCmd->Cdw10 = 1;\r
  CommandPacket.NvmeCmd->Flags = CDW10_VALID;\r
\r
  Status = NvmePassThruExecute (\r
             Private,\r
             NVME_CONTROLLER_NSID,\r
             &CommandPacket\r
             );\r
  return Status;\r
}\r
\r
/**\r
  Get specified identify namespace data.\r
\r
  @param[in] Private        The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.\r
  @param[in] NamespaceId    The specified namespace identifier.\r
  @param[in] Buffer         The buffer used to store the identify namespace data.\r
\r
  @return EFI_SUCCESS         Successfully get the identify namespace data.\r
  @return EFI_DEVICE_ERROR    Fail to get the identify namespace data.\r
\r
**/\r
EFI_STATUS\r
NvmeIdentifyNamespace (\r
  IN PEI_NVME_CONTROLLER_PRIVATE_DATA  *Private,\r
  IN UINT32                            NamespaceId,\r
  IN VOID                              *Buffer\r
  )\r
{\r
  EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET  CommandPacket;\r
  EFI_NVM_EXPRESS_COMMAND                   Command;\r
  EFI_NVM_EXPRESS_COMPLETION                Completion;\r
  EFI_STATUS                                Status;\r
\r
  ZeroMem (&CommandPacket, sizeof (EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));\r
  ZeroMem (&Command, sizeof (EFI_NVM_EXPRESS_COMMAND));\r
  ZeroMem (&Completion, sizeof (EFI_NVM_EXPRESS_COMPLETION));\r
\r
  Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD;\r
  Command.Nsid        = NamespaceId;\r
\r
  CommandPacket.NvmeCmd        = &Command;\r
  CommandPacket.NvmeCompletion = &Completion;\r
  CommandPacket.TransferBuffer = Buffer;\r
  CommandPacket.TransferLength = sizeof (NVME_ADMIN_NAMESPACE_DATA);\r
  CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;\r
  CommandPacket.QueueType      = NVME_ADMIN_QUEUE;\r
  //\r
  // Set bit 0 (Cns bit) to 1 to identify a namespace\r
  //\r
  CommandPacket.NvmeCmd->Cdw10 = 0;\r
  CommandPacket.NvmeCmd->Flags = CDW10_VALID;\r
\r
  Status = NvmePassThruExecute (\r
             Private,\r
             NamespaceId,\r
             &CommandPacket\r
             );\r
  return Status;\r
}\r
\r
/**\r
  Dump the Identify Controller data.\r
\r
  @param[in] ControllerData    The pointer to the NVME_ADMIN_CONTROLLER_DATA data structure.\r
\r
**/\r
VOID\r
NvmeDumpControllerData (\r
  IN NVME_ADMIN_CONTROLLER_DATA  *ControllerData\r
  )\r
{\r
  UINT8  Sn[21];\r
  UINT8  Mn[41];\r
\r
  CopyMem (Sn, ControllerData->Sn, sizeof (ControllerData->Sn));\r
  Sn[20] = 0;\r
  CopyMem (Mn, ControllerData->Mn, sizeof (ControllerData->Mn));\r
  Mn[40] = 0;\r
\r
  DEBUG ((DEBUG_INFO, " == NVME IDENTIFY CONTROLLER DATA ==\n"));\r
  DEBUG ((DEBUG_INFO, "    PCI VID   : 0x%x\n", ControllerData->Vid));\r
  DEBUG ((DEBUG_INFO, "    PCI SSVID : 0x%x\n", ControllerData->Ssvid));\r
  DEBUG ((DEBUG_INFO, "    SN        : %a\n", Sn));\r
  DEBUG ((DEBUG_INFO, "    MN        : %a\n", Mn));\r
  DEBUG ((DEBUG_INFO, "    FR        : 0x%lx\n", *((UINT64 *)ControllerData->Fr)));\r
  DEBUG ((DEBUG_INFO, "    RAB       : 0x%x\n", ControllerData->Rab));\r
  DEBUG ((DEBUG_INFO, "    IEEE      : 0x%x\n", *(UINT32 *)ControllerData->Ieee_oui));\r
  DEBUG ((DEBUG_INFO, "    AERL      : 0x%x\n", ControllerData->Aerl));\r
  DEBUG ((DEBUG_INFO, "    SQES      : 0x%x\n", ControllerData->Sqes));\r
  DEBUG ((DEBUG_INFO, "    CQES      : 0x%x\n", ControllerData->Cqes));\r
  DEBUG ((DEBUG_INFO, "    NN        : 0x%x\n", ControllerData->Nn));\r
  return;\r
}\r
\r
/**\r
  Create IO completion queue.\r
\r
  @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.\r
\r
  @return EFI_SUCCESS    Successfully create io completion queue.\r
  @return others         Fail to create io completion queue.\r
\r
**/\r
EFI_STATUS\r
NvmeCreateIoCompletionQueue (\r
  IN PEI_NVME_CONTROLLER_PRIVATE_DATA  *Private\r
  )\r
{\r
  EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET  CommandPacket;\r
  EFI_NVM_EXPRESS_COMMAND                   Command;\r
  EFI_NVM_EXPRESS_COMPLETION                Completion;\r
  EFI_STATUS                                Status;\r
  NVME_ADMIN_CRIOCQ                         CrIoCq;\r
\r
  ZeroMem (&CommandPacket, sizeof (EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));\r
  ZeroMem (&Command, sizeof (EFI_NVM_EXPRESS_COMMAND));\r
  ZeroMem (&Completion, sizeof (EFI_NVM_EXPRESS_COMPLETION));\r
  ZeroMem (&CrIoCq, sizeof (NVME_ADMIN_CRIOCQ));\r
\r
  CommandPacket.NvmeCmd        = &Command;\r
  CommandPacket.NvmeCompletion = &Completion;\r
\r
  Command.Cdw0.Opcode          = NVME_ADMIN_CRIOCQ_CMD;\r
  CommandPacket.TransferBuffer = Private->CqBuffer[NVME_IO_QUEUE];\r
  CommandPacket.TransferLength = EFI_PAGE_SIZE;\r
  CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;\r
  CommandPacket.QueueType      = NVME_ADMIN_QUEUE;\r
\r
  CrIoCq.Qid   = NVME_IO_QUEUE;\r
  CrIoCq.Qsize = NVME_CCQ_SIZE;\r
  CrIoCq.Pc    = 1;\r
  CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoCq, sizeof (NVME_ADMIN_CRIOCQ));\r
  CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;\r
\r
  Status = NvmePassThruExecute (\r
             Private,\r
             NVME_CONTROLLER_NSID,\r
             &CommandPacket\r
             );\r
  return Status;\r
}\r
\r
/**\r
  Create IO submission queue.\r
\r
  @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.\r
\r
  @return EFI_SUCCESS    Successfully create io submission queue.\r
  @return others         Fail to create io submission queue.\r
\r
**/\r
EFI_STATUS\r
NvmeCreateIoSubmissionQueue (\r
  IN PEI_NVME_CONTROLLER_PRIVATE_DATA  *Private\r
  )\r
{\r
  EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET  CommandPacket;\r
  EFI_NVM_EXPRESS_COMMAND                   Command;\r
  EFI_NVM_EXPRESS_COMPLETION                Completion;\r
  EFI_STATUS                                Status;\r
  NVME_ADMIN_CRIOSQ                         CrIoSq;\r
\r
  ZeroMem (&CommandPacket, sizeof (EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));\r
  ZeroMem (&Command, sizeof (EFI_NVM_EXPRESS_COMMAND));\r
  ZeroMem (&Completion, sizeof (EFI_NVM_EXPRESS_COMPLETION));\r
  ZeroMem (&CrIoSq, sizeof (NVME_ADMIN_CRIOSQ));\r
\r
  CommandPacket.NvmeCmd        = &Command;\r
  CommandPacket.NvmeCompletion = &Completion;\r
\r
  Command.Cdw0.Opcode          = NVME_ADMIN_CRIOSQ_CMD;\r
  CommandPacket.TransferBuffer = Private->SqBuffer[NVME_IO_QUEUE];\r
  CommandPacket.TransferLength = EFI_PAGE_SIZE;\r
  CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;\r
  CommandPacket.QueueType      = NVME_ADMIN_QUEUE;\r
\r
  CrIoSq.Qid   = NVME_IO_QUEUE;\r
  CrIoSq.Qsize = NVME_CSQ_SIZE;\r
  CrIoSq.Pc    = 1;\r
  CrIoSq.Cqid  = NVME_IO_QUEUE;\r
  CrIoSq.Qprio = 0;\r
  CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoSq, sizeof (NVME_ADMIN_CRIOSQ));\r
  CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;\r
\r
  Status = NvmePassThruExecute (\r
             Private,\r
             NVME_CONTROLLER_NSID,\r
             &CommandPacket\r
             );\r
  return Status;\r
}\r
\r
/**\r
  Initialize the Nvm Express controller.\r
\r
  @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.\r
\r
  @retval EFI_SUCCESS    The NVM Express Controller is initialized successfully.\r
  @retval Others         A device error occurred while initializing the controller.\r
\r
**/\r
EFI_STATUS\r
NvmeControllerInit (\r
  IN PEI_NVME_CONTROLLER_PRIVATE_DATA  *Private\r
  )\r
{\r
  EFI_STATUS  Status;\r
  UINTN       Index;\r
  NVME_AQA    Aqa;\r
  NVME_ASQ    Asq;\r
  NVME_ACQ    Acq;\r
  NVME_VER    Ver;\r
\r
  //\r
  // Dump the NVME controller implementation version\r
  //\r
  NVME_GET_VER (Private, &Ver);\r
  DEBUG ((DEBUG_INFO, "NVME controller implementation version: %d.%d\n", Ver.Mjr, Ver.Mnr));\r
\r
  //\r
  // Read the controller Capabilities register and verify that the NVM command set is supported\r
  //\r
  NVME_GET_CAP (Private, &Private->Cap);\r
  if (Private->Cap.Css != 0x01) {\r
    DEBUG ((DEBUG_ERROR, "%a: The NVME controller doesn't support NVMe command set.\n", __FUNCTION__));\r
    return EFI_UNSUPPORTED;\r
  }\r
\r
  //\r
  // Currently, the driver only supports 4k page size\r
  //\r
  if ((Private->Cap.Mpsmin + 12) > EFI_PAGE_SHIFT) {\r
    DEBUG ((DEBUG_ERROR, "%a: The driver doesn't support page size other than 4K.\n", __FUNCTION__));\r
    ASSERT (FALSE);\r
    return EFI_UNSUPPORTED;\r
  }\r
\r
  for (Index = 0; Index < NVME_MAX_QUEUES; Index++) {\r
    Private->Pt[Index]  = 0;\r
    Private->Cid[Index] = 0;\r
    ZeroMem ((VOID *)(UINTN)(&Private->SqTdbl[Index]), sizeof (NVME_SQTDBL));\r
    ZeroMem ((VOID *)(UINTN)(&Private->CqHdbl[Index]), sizeof (NVME_CQHDBL));\r
  }\r
\r
  ZeroMem (Private->Buffer, EFI_PAGE_SIZE * NVME_MEM_MAX_PAGES);\r
\r
  //\r
  // Disable the NVME controller first\r
  //\r
  Status = NvmeDisableController (Private);\r
  if (EFI_ERROR (Status)) {\r
    DEBUG ((DEBUG_ERROR, "%a: NvmeDisableController fail, Status - %r\n", __FUNCTION__, Status));\r
    return Status;\r
  }\r
\r
  //\r
  // Set the number of entries in admin submission & completion queues\r
  //\r
  Aqa.Asqs  = NVME_ASQ_SIZE;\r
  Aqa.Rsvd1 = 0;\r
  Aqa.Acqs  = NVME_ACQ_SIZE;\r
  Aqa.Rsvd2 = 0;\r
\r
  //\r
  // Address of admin submission & completion queues\r
  //\r
  Asq = (UINT64)(UINTN)(NVME_ASQ_BASE (Private) & ~0xFFF);\r
  Acq = (UINT64)(UINTN)(NVME_ACQ_BASE (Private) & ~0xFFF);\r
\r
  //\r
  // Address of I/O submission & completion queues\r
  //\r
  Private->SqBuffer[0] = (NVME_SQ *)(UINTN)NVME_ASQ_BASE (Private);    // NVME_ADMIN_QUEUE\r
  Private->CqBuffer[0] = (NVME_CQ *)(UINTN)NVME_ACQ_BASE (Private);    // NVME_ADMIN_QUEUE\r
  Private->SqBuffer[1] = (NVME_SQ *)(UINTN)NVME_SQ_BASE (Private, 0);  // NVME_IO_QUEUE\r
  Private->CqBuffer[1] = (NVME_CQ *)(UINTN)NVME_CQ_BASE (Private, 0);  // NVME_IO_QUEUE\r
  DEBUG ((DEBUG_INFO, "Admin Submission Queue Size (Aqa.Asqs) = [%08X]\n", Aqa.Asqs));\r
  DEBUG ((DEBUG_INFO, "Admin Completion Queue Size (Aqa.Acqs) = [%08X]\n", Aqa.Acqs));\r
  DEBUG ((DEBUG_INFO, "Admin Submission Queue (SqBuffer[0]) =   [%08X]\n", Private->SqBuffer[0]));\r
  DEBUG ((DEBUG_INFO, "Admin Completion Queue (CqBuffer[0]) =   [%08X]\n", Private->CqBuffer[0]));\r
  DEBUG ((DEBUG_INFO, "I/O   Submission Queue (SqBuffer[1]) =   [%08X]\n", Private->SqBuffer[1]));\r
  DEBUG ((DEBUG_INFO, "I/O   Completion Queue (CqBuffer[1]) =   [%08X]\n", Private->CqBuffer[1]));\r
\r
  //\r
  // Program admin queue attributes\r
  //\r
  NVME_SET_AQA (Private, &Aqa);\r
\r
  //\r
  // Program admin submission & completion queues address\r
  //\r
  NVME_SET_ASQ (Private, &Asq);\r
  NVME_SET_ACQ (Private, &Acq);\r
\r
  //\r
  // Enable the NVME controller\r
  //\r
  Status = NvmeEnableController (Private);\r
  if (EFI_ERROR (Status)) {\r
    DEBUG ((DEBUG_ERROR, "%a: NvmeEnableController fail, Status - %r\n", __FUNCTION__, Status));\r
    return Status;\r
  }\r
\r
  //\r
  // Get the Identify Controller data\r
  //\r
  if (Private->ControllerData == NULL) {\r
    Private->ControllerData = (NVME_ADMIN_CONTROLLER_DATA *)AllocateZeroPool (sizeof (NVME_ADMIN_CONTROLLER_DATA));\r
    if (Private->ControllerData == NULL) {\r
      return EFI_OUT_OF_RESOURCES;\r
    }\r
  }\r
\r
  Status = NvmeIdentifyController (Private, Private->ControllerData);\r
  if (EFI_ERROR (Status)) {\r
    DEBUG ((DEBUG_ERROR, "%a: NvmeIdentifyController fail, Status - %r\n", __FUNCTION__, Status));\r
    return Status;\r
  }\r
\r
  NvmeDumpControllerData (Private->ControllerData);\r
\r
  //\r
  // Check the namespace number for storing the namespaces information\r
  //\r
  if (Private->ControllerData->Nn > MAX_UINT32 / sizeof (PEI_NVME_NAMESPACE_INFO)) {\r
    DEBUG ((\r
      DEBUG_ERROR,\r
      "%a: Number of Namespaces field in Identify Controller data not supported by the driver.\n",\r
      __FUNCTION__\r
      ));\r
    return EFI_UNSUPPORTED;\r
  }\r
\r
  //\r
  // Create one I/O completion queue and one I/O submission queue\r
  //\r
  Status = NvmeCreateIoCompletionQueue (Private);\r
  if (EFI_ERROR (Status)) {\r
    DEBUG ((DEBUG_ERROR, "%a: Create IO completion queue fail, Status - %r\n", __FUNCTION__, Status));\r
    return Status;\r
  }\r
\r
  Status = NvmeCreateIoSubmissionQueue (Private);\r
  if (EFI_ERROR (Status)) {\r
    DEBUG ((DEBUG_ERROR, "%a: Create IO submission queue fail, Status - %r\n", __FUNCTION__, Status));\r
  }\r
\r
  return Status;\r
}\r
\r
/**\r
  Free the DMA resources allocated by an NVME controller.\r
\r
  @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.\r
\r
**/\r
VOID\r
NvmeFreeDmaResource (\r
  IN PEI_NVME_CONTROLLER_PRIVATE_DATA  *Private\r
  )\r
{\r
  ASSERT (Private != NULL);\r
\r
  if (Private->BufferMapping != NULL) {\r
    IoMmuFreeBuffer (\r
      NVME_MEM_MAX_PAGES,\r
      Private->Buffer,\r
      Private->BufferMapping\r
      );\r
  }\r
\r
  return;\r
}\r