[mirror_edk2.git] / MdeModulePkg / Bus / Pci / NvmExpressDxe / NvmExpressHci.c

/** @file\r
  NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows\r
  NVM Express specification.\r
\r
  Copyright (c) 2013 - 2019, Intel Corporation. All rights reserved.<BR>\r
  SPDX-License-Identifier: BSD-2-Clause-Patent\r
\r
**/\r
\r
#include "NvmExpress.h"\r
\r
#define NVME_SHUTDOWN_PROCESS_TIMEOUT 45\r
\r
//\r
// The number of NVME controllers managed by this driver, used by\r
// NvmeRegisterShutdownNotification() and NvmeUnregisterShutdownNotification().\r
//\r
UINTN                           mNvmeControllerNumber = 0;\r
\r
/**\r
  Read Nvm Express controller capability register.\r
\r
  @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.\r
  @param  Cap              The buffer used to store capability register content.\r
\r
  @return EFI_SUCCESS      Successfully read the controller capability register content.\r
  @return EFI_DEVICE_ERROR Fail to read the controller capability register.\r
\r
**/\r
EFI_STATUS\r
ReadNvmeControllerCapabilities (\r
  IN NVME_CONTROLLER_PRIVATE_DATA     *Private,\r
  IN NVME_CAP                         *Cap\r
  )\r
{\r
  EFI_PCI_IO_PROTOCOL   *PciIo;\r
  EFI_STATUS            Status;\r
  UINT64                Data;\r
\r
  PciIo  = Private->PciIo;\r
  Status = PciIo->Mem.Read (\r
                        PciIo,\r
                        EfiPciIoWidthUint32,\r
                        NVME_BAR,\r
                        NVME_CAP_OFFSET,\r
                        2,\r
                        &Data\r
                        );\r
\r
  if (EFI_ERROR(Status)) {\r
    return Status;\r
  }\r
\r
  WriteUnaligned64 ((UINT64*)Cap, Data);\r
  return EFI_SUCCESS;\r
}\r
\r
/**\r
  Read Nvm Express controller configuration register.\r
\r
  @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.\r
  @param  Cc               The buffer used to store configuration register content.\r
\r
  @return EFI_SUCCESS      Successfully read the controller configuration register content.\r
  @return EFI_DEVICE_ERROR Fail to read the controller configuration register.\r
\r
**/\r
EFI_STATUS\r
ReadNvmeControllerConfiguration (\r
  IN NVME_CONTROLLER_PRIVATE_DATA     *Private,\r
  IN NVME_CC                          *Cc\r
  )\r
{\r
  EFI_PCI_IO_PROTOCOL   *PciIo;\r
  EFI_STATUS            Status;\r
  UINT32                Data;\r
\r
  PciIo  = Private->PciIo;\r
  Status = PciIo->Mem.Read (\r
                        PciIo,\r
                        EfiPciIoWidthUint32,\r
                        NVME_BAR,\r
                        NVME_CC_OFFSET,\r
                        1,\r
                        &Data\r
                        );\r
\r
  if (EFI_ERROR(Status)) {\r
    return Status;\r
  }\r
\r
  WriteUnaligned32 ((UINT32*)Cc, Data);\r
  return EFI_SUCCESS;\r
}\r
\r
/**\r
  Write Nvm Express controller configuration register.\r
\r
  @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.\r
  @param  Cc               The buffer used to store the content to be written into configuration register.\r
\r
  @return EFI_SUCCESS      Successfully write data into the controller configuration register.\r
  @return EFI_DEVICE_ERROR Fail to write data into the controller configuration register.\r
\r
**/\r
EFI_STATUS\r
WriteNvmeControllerConfiguration (\r
  IN NVME_CONTROLLER_PRIVATE_DATA     *Private,\r
  IN NVME_CC                          *Cc\r
  )\r
{\r
  EFI_PCI_IO_PROTOCOL   *PciIo;\r
  EFI_STATUS            Status;\r
  UINT32                Data;\r
\r
  PciIo  = Private->PciIo;\r
  Data   = ReadUnaligned32 ((UINT32*)Cc);\r
  Status = PciIo->Mem.Write (\r
                        PciIo,\r
                        EfiPciIoWidthUint32,\r
                        NVME_BAR,\r
                        NVME_CC_OFFSET,\r
                        1,\r
                        &Data\r
                        );\r
\r
  if (EFI_ERROR(Status)) {\r
    return Status;\r
  }\r
\r
  DEBUG ((EFI_D_INFO, "Cc.En: %d\n", Cc->En));\r
  DEBUG ((EFI_D_INFO, "Cc.Css: %d\n", Cc->Css));\r
  DEBUG ((EFI_D_INFO, "Cc.Mps: %d\n", Cc->Mps));\r
  DEBUG ((EFI_D_INFO, "Cc.Ams: %d\n", Cc->Ams));\r
  DEBUG ((EFI_D_INFO, "Cc.Shn: %d\n", Cc->Shn));\r
  DEBUG ((EFI_D_INFO, "Cc.Iosqes: %d\n", Cc->Iosqes));\r
  DEBUG ((EFI_D_INFO, "Cc.Iocqes: %d\n", Cc->Iocqes));\r
\r
  return EFI_SUCCESS;\r
}\r
\r
/**\r
  Read Nvm Express controller status register.\r
\r
  @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.\r
  @param  Csts             The buffer used to store status register content.\r
\r
  @return EFI_SUCCESS      Successfully read the controller status register content.\r
  @return EFI_DEVICE_ERROR Fail to read the controller status register.\r
\r
**/\r
EFI_STATUS\r
ReadNvmeControllerStatus (\r
  IN NVME_CONTROLLER_PRIVATE_DATA     *Private,\r
  IN NVME_CSTS                        *Csts\r
  )\r
{\r
  EFI_PCI_IO_PROTOCOL   *PciIo;\r
  EFI_STATUS            Status;\r
  UINT32                Data;\r
\r
  PciIo  = Private->PciIo;\r
  Status = PciIo->Mem.Read (\r
                        PciIo,\r
                        EfiPciIoWidthUint32,\r
                        NVME_BAR,\r
                        NVME_CSTS_OFFSET,\r
                        1,\r
                        &Data\r
                        );\r
\r
  if (EFI_ERROR(Status)) {\r
    return Status;\r
  }\r
\r
  WriteUnaligned32 ((UINT32*)Csts, Data);\r
  return EFI_SUCCESS;\r
}\r
\r
\r
\r
/**\r
  Write Nvm Express admin queue attributes register.\r
\r
  @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.\r
  @param  Aqa              The buffer used to store the content to be written into admin queue attributes register.\r
\r
  @return EFI_SUCCESS      Successfully write data into the admin queue attributes register.\r
  @return EFI_DEVICE_ERROR Fail to write data into the admin queue attributes register.\r
\r
**/\r
EFI_STATUS\r
WriteNvmeAdminQueueAttributes (\r
  IN NVME_CONTROLLER_PRIVATE_DATA     *Private,\r
  IN NVME_AQA                         *Aqa\r
  )\r
{\r
  EFI_PCI_IO_PROTOCOL   *PciIo;\r
  EFI_STATUS            Status;\r
  UINT32                Data;\r
\r
  PciIo  = Private->PciIo;\r
  Data   = ReadUnaligned32 ((UINT32*)Aqa);\r
  Status = PciIo->Mem.Write (\r
                        PciIo,\r
                        EfiPciIoWidthUint32,\r
                        NVME_BAR,\r
                        NVME_AQA_OFFSET,\r
                        1,\r
                        &Data\r
                        );\r
\r
  if (EFI_ERROR(Status)) {\r
    return Status;\r
  }\r
\r
  DEBUG ((EFI_D_INFO, "Aqa.Asqs: %d\n", Aqa->Asqs));\r
  DEBUG ((EFI_D_INFO, "Aqa.Acqs: %d\n", Aqa->Acqs));\r
\r
  return EFI_SUCCESS;\r
}\r
\r
\r
/**\r
  Write Nvm Express admin submission queue base address register.\r
\r
  @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.\r
  @param  Asq              The buffer used to store the content to be written into admin submission queue base address register.\r
\r
  @return EFI_SUCCESS      Successfully write data into the admin submission queue base address register.\r
  @return EFI_DEVICE_ERROR Fail to write data into the admin submission queue base address register.\r
\r
**/\r
EFI_STATUS\r
WriteNvmeAdminSubmissionQueueBaseAddress (\r
  IN NVME_CONTROLLER_PRIVATE_DATA     *Private,\r
  IN NVME_ASQ                         *Asq\r
  )\r
{\r
  EFI_PCI_IO_PROTOCOL   *PciIo;\r
  EFI_STATUS            Status;\r
  UINT64                Data;\r
\r
  PciIo  = Private->PciIo;\r
  Data   = ReadUnaligned64 ((UINT64*)Asq);\r
\r
  Status = PciIo->Mem.Write (\r
                        PciIo,\r
                        EfiPciIoWidthUint32,\r
                        NVME_BAR,\r
                        NVME_ASQ_OFFSET,\r
                        2,\r
                        &Data\r
                        );\r
\r
  if (EFI_ERROR(Status)) {\r
    return Status;\r
  }\r
\r
  DEBUG ((EFI_D_INFO, "Asq: %lx\n", *Asq));\r
\r
  return EFI_SUCCESS;\r
}\r
\r
\r
\r
/**\r
  Write Nvm Express admin completion queue base address register.\r
\r
  @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.\r
  @param  Acq              The buffer used to store the content to be written into admin completion queue base address register.\r
\r
  @return EFI_SUCCESS      Successfully write data into the admin completion queue base address register.\r
  @return EFI_DEVICE_ERROR Fail to write data into the admin completion queue base address register.\r
\r
**/\r
EFI_STATUS\r
WriteNvmeAdminCompletionQueueBaseAddress (\r
  IN NVME_CONTROLLER_PRIVATE_DATA     *Private,\r
  IN NVME_ACQ                         *Acq\r
  )\r
{\r
  EFI_PCI_IO_PROTOCOL   *PciIo;\r
  EFI_STATUS            Status;\r
  UINT64                Data;\r
\r
  PciIo  = Private->PciIo;\r
  Data   = ReadUnaligned64 ((UINT64*)Acq);\r
\r
  Status = PciIo->Mem.Write (\r
                        PciIo,\r
                        EfiPciIoWidthUint32,\r
                        NVME_BAR,\r
                        NVME_ACQ_OFFSET,\r
                        2,\r
                        &Data\r
                        );\r
\r
  if (EFI_ERROR(Status)) {\r
    return Status;\r
  }\r
\r
  DEBUG ((EFI_D_INFO, "Acq: %lxh\n", *Acq));\r
\r
  return EFI_SUCCESS;\r
}\r
\r
/**\r
  Disable the Nvm Express controller.\r
\r
  @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.\r
\r
  @return EFI_SUCCESS      Successfully disable the controller.\r
  @return EFI_DEVICE_ERROR Fail to disable the controller.\r
\r
**/\r
EFI_STATUS\r
NvmeDisableController (\r
  IN NVME_CONTROLLER_PRIVATE_DATA     *Private\r
  )\r
{\r
  NVME_CC                Cc;\r
  NVME_CSTS              Csts;\r
  EFI_STATUS             Status;\r
  UINT32                 Index;\r
  UINT8                  Timeout;\r
\r
  //\r
  // Read Controller Configuration Register.\r
  //\r
  Status = ReadNvmeControllerConfiguration (Private, &Cc);\r
  if (EFI_ERROR(Status)) {\r
    return Status;\r
  }\r
\r
  Cc.En = 0;\r
\r
  //\r
  // Disable the controller.\r
  //\r
  Status = WriteNvmeControllerConfiguration (Private, &Cc);\r
\r
  if (EFI_ERROR(Status)) {\r
    return Status;\r
  }\r
\r
  //\r
  // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to transition from 1 to 0 after\r
  // Cc.Enable transition from 1 to 0. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To.\r
  //\r
  if (Private->Cap.To == 0) {\r
    Timeout = 1;\r
  } else {\r
    Timeout = Private->Cap.To;\r
  }\r
\r
  for(Index = (Timeout * 500); Index != 0; --Index) {\r
    gBS->Stall(1000);\r
\r
    //\r
    // Check if the controller is initialized\r
    //\r
    Status = ReadNvmeControllerStatus (Private, &Csts);\r
\r
    if (EFI_ERROR(Status)) {\r
      return Status;\r
    }\r
\r
    if (Csts.Rdy == 0) {\r
      break;\r
    }\r
  }\r
\r
  if (Index == 0) {\r
    Status = EFI_DEVICE_ERROR;\r
    REPORT_STATUS_CODE (\r
      (EFI_ERROR_CODE | EFI_ERROR_MAJOR),\r
      (EFI_IO_BUS_SCSI | EFI_IOB_EC_INTERFACE_ERROR)\r
      );\r
  }\r
\r
  DEBUG ((EFI_D_INFO, "NVMe controller is disabled with status [%r].\n", Status));\r
  return Status;\r
}\r
\r
/**\r
  Enable the Nvm Express controller.\r
\r
  @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.\r
\r
  @return EFI_SUCCESS      Successfully enable the controller.\r
  @return EFI_DEVICE_ERROR Fail to enable the controller.\r
  @return EFI_TIMEOUT      Fail to enable the controller in given time slot.\r
\r
**/\r
EFI_STATUS\r
NvmeEnableController (\r
  IN NVME_CONTROLLER_PRIVATE_DATA     *Private\r
  )\r
{\r
  NVME_CC                Cc;\r
  NVME_CSTS              Csts;\r
  EFI_STATUS             Status;\r
  UINT32                 Index;\r
  UINT8                  Timeout;\r
\r
  //\r
  // Enable the controller.\r
  // CC.AMS, CC.MPS and CC.CSS are all set to 0.\r
  //\r
  ZeroMem (&Cc, sizeof (NVME_CC));\r
  Cc.En     = 1;\r
  Cc.Iosqes = 6;\r
  Cc.Iocqes = 4;\r
\r
  Status = WriteNvmeControllerConfiguration (Private, &Cc);\r
  if (EFI_ERROR(Status)) {\r
    return Status;\r
  }\r
\r
  //\r
  // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to set after\r
  // Cc.Enable. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To.\r
  //\r
  if (Private->Cap.To == 0) {\r
    Timeout = 1;\r
  } else {\r
    Timeout = Private->Cap.To;\r
  }\r
\r
  for(Index = (Timeout * 500); Index != 0; --Index) {\r
    gBS->Stall(1000);\r
\r
    //\r
    // Check if the controller is initialized\r
    //\r
    Status = ReadNvmeControllerStatus (Private, &Csts);\r
\r
    if (EFI_ERROR(Status)) {\r
      return Status;\r
    }\r
\r
    if (Csts.Rdy) {\r
      break;\r
    }\r
  }\r
\r
  if (Index == 0) {\r
    Status = EFI_TIMEOUT;\r
    REPORT_STATUS_CODE (\r
      (EFI_ERROR_CODE | EFI_ERROR_MAJOR),\r
      (EFI_IO_BUS_SCSI | EFI_IOB_EC_INTERFACE_ERROR)\r
      );\r
  }\r
\r
  DEBUG ((EFI_D_INFO, "NVMe controller is enabled with status [%r].\n", Status));\r
  return Status;\r
}\r
\r
/**\r
  Get identify controller data.\r
\r
  @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.\r
  @param  Buffer           The buffer used to store the identify controller data.\r
\r
  @return EFI_SUCCESS      Successfully get the identify controller data.\r
  @return EFI_DEVICE_ERROR Fail to get the identify controller data.\r
\r
**/\r
EFI_STATUS\r
NvmeIdentifyController (\r
  IN NVME_CONTROLLER_PRIVATE_DATA       *Private,\r
  IN VOID                               *Buffer\r
  )\r
{\r
  EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;\r
  EFI_NVM_EXPRESS_COMMAND                  Command;\r
  EFI_NVM_EXPRESS_COMPLETION               Completion;\r
  EFI_STATUS                               Status;\r
\r
  ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));\r
  ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND));\r
  ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION));\r
\r
  Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD;\r
  //\r
  // According to Nvm Express 1.1 spec Figure 38, When not used, the field shall be cleared to 0h.\r
  // For the Identify command, the Namespace Identifier is only used for the Namespace data structure.\r
  //\r
  Command.Nsid        = 0;\r
\r
  CommandPacket.NvmeCmd        = &Command;\r
  CommandPacket.NvmeCompletion = &Completion;\r
  CommandPacket.TransferBuffer = Buffer;\r
  CommandPacket.TransferLength = sizeof (NVME_ADMIN_CONTROLLER_DATA);\r
  CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;\r
  CommandPacket.QueueType      = NVME_ADMIN_QUEUE;\r
  //\r
  // Set bit 0 (Cns bit) to 1 to identify a controller\r
  //\r
  Command.Cdw10                = 1;\r
  Command.Flags                = CDW10_VALID;\r
\r
  Status = Private->Passthru.PassThru (\r
                               &Private->Passthru,\r
                               NVME_CONTROLLER_ID,\r
                               &CommandPacket,\r
                               NULL\r
                               );\r
\r
  return Status;\r
}\r
\r
/**\r
  Get specified identify namespace data.\r
\r
  @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.\r
  @param  NamespaceId      The specified namespace identifier.\r
  @param  Buffer           The buffer used to store the identify namespace data.\r
\r
  @return EFI_SUCCESS      Successfully get the identify namespace data.\r
  @return EFI_DEVICE_ERROR Fail to get the identify namespace data.\r
\r
**/\r
EFI_STATUS\r
NvmeIdentifyNamespace (\r
  IN NVME_CONTROLLER_PRIVATE_DATA      *Private,\r
  IN UINT32                            NamespaceId,\r
  IN VOID                              *Buffer\r
  )\r
{\r
  EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;\r
  EFI_NVM_EXPRESS_COMMAND                  Command;\r
  EFI_NVM_EXPRESS_COMPLETION               Completion;\r
  EFI_STATUS                               Status;\r
\r
  ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));\r
  ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND));\r
  ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION));\r
\r
  CommandPacket.NvmeCmd        = &Command;\r
  CommandPacket.NvmeCompletion = &Completion;\r
\r
  Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD;\r
  Command.Nsid        = NamespaceId;\r
  CommandPacket.TransferBuffer = Buffer;\r
  CommandPacket.TransferLength = sizeof (NVME_ADMIN_NAMESPACE_DATA);\r
  CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;\r
  CommandPacket.QueueType      = NVME_ADMIN_QUEUE;\r
  //\r
  // Set bit 0 (Cns bit) to 1 to identify a namespace\r
  //\r
  CommandPacket.NvmeCmd->Cdw10 = 0;\r
  CommandPacket.NvmeCmd->Flags = CDW10_VALID;\r
\r
  Status = Private->Passthru.PassThru (\r
                               &Private->Passthru,\r
                               NamespaceId,\r
                               &CommandPacket,\r
                               NULL\r
                               );\r
\r
  return Status;\r
}\r
\r
/**\r
  Create io completion queue.\r
\r
  @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.\r
\r
  @return EFI_SUCCESS      Successfully create io completion queue.\r
  @return EFI_DEVICE_ERROR Fail to create io completion queue.\r
\r
**/\r
EFI_STATUS\r
NvmeCreateIoCompletionQueue (\r
  IN NVME_CONTROLLER_PRIVATE_DATA      *Private\r
  )\r
{\r
  EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;\r
  EFI_NVM_EXPRESS_COMMAND                  Command;\r
  EFI_NVM_EXPRESS_COMPLETION               Completion;\r
  EFI_STATUS                               Status;\r
  NVME_ADMIN_CRIOCQ                        CrIoCq;\r
  UINT32                                   Index;\r
  UINT16                                   QueueSize;\r
\r
  Status = EFI_SUCCESS;\r
  Private->CreateIoQueue = TRUE;\r
\r
  for (Index = 1; Index < NVME_MAX_QUEUES; Index++) {\r
    ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));\r
    ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND));\r
    ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION));\r
    ZeroMem (&CrIoCq, sizeof(NVME_ADMIN_CRIOCQ));\r
\r
    CommandPacket.NvmeCmd        = &Command;\r
    CommandPacket.NvmeCompletion = &Completion;\r
\r
    Command.Cdw0.Opcode = NVME_ADMIN_CRIOCQ_CMD;\r
    CommandPacket.TransferBuffer = Private->CqBufferPciAddr[Index];\r
    CommandPacket.TransferLength = EFI_PAGE_SIZE;\r
    CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;\r
    CommandPacket.QueueType      = NVME_ADMIN_QUEUE;\r
\r
    if (Index == 1) {\r
      QueueSize = NVME_CCQ_SIZE;\r
    } else {\r
      if (Private->Cap.Mqes > NVME_ASYNC_CCQ_SIZE) {\r
        QueueSize = NVME_ASYNC_CCQ_SIZE;\r
      } else {\r
        QueueSize = Private->Cap.Mqes;\r
      }\r
    }\r
\r
    CrIoCq.Qid   = Index;\r
    CrIoCq.Qsize = QueueSize;\r
    CrIoCq.Pc    = 1;\r
    CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoCq, sizeof (NVME_ADMIN_CRIOCQ));\r
    CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;\r
\r
    Status = Private->Passthru.PassThru (\r
                                 &Private->Passthru,\r
                                 0,\r
                                 &CommandPacket,\r
                                 NULL\r
                                 );\r
    if (EFI_ERROR (Status)) {\r
      break;\r
    }\r
  }\r
\r
  Private->CreateIoQueue = FALSE;\r
\r
  return Status;\r
}\r
\r
/**\r
  Create io submission queue.\r
\r
  @param  Private          The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.\r
\r
  @return EFI_SUCCESS      Successfully create io submission queue.\r
  @return EFI_DEVICE_ERROR Fail to create io submission queue.\r
\r
**/\r
EFI_STATUS\r
NvmeCreateIoSubmissionQueue (\r
  IN NVME_CONTROLLER_PRIVATE_DATA      *Private\r
  )\r
{\r
  EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;\r
  EFI_NVM_EXPRESS_COMMAND                  Command;\r
  EFI_NVM_EXPRESS_COMPLETION               Completion;\r
  EFI_STATUS                               Status;\r
  NVME_ADMIN_CRIOSQ                        CrIoSq;\r
  UINT32                                   Index;\r
  UINT16                                   QueueSize;\r
\r
  Status = EFI_SUCCESS;\r
  Private->CreateIoQueue = TRUE;\r
\r
  for (Index = 1; Index < NVME_MAX_QUEUES; Index++) {\r
    ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));\r
    ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND));\r
    ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION));\r
    ZeroMem (&CrIoSq, sizeof(NVME_ADMIN_CRIOSQ));\r
\r
    CommandPacket.NvmeCmd        = &Command;\r
    CommandPacket.NvmeCompletion = &Completion;\r
\r
    Command.Cdw0.Opcode = NVME_ADMIN_CRIOSQ_CMD;\r
    CommandPacket.TransferBuffer = Private->SqBufferPciAddr[Index];\r
    CommandPacket.TransferLength = EFI_PAGE_SIZE;\r
    CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;\r
    CommandPacket.QueueType      = NVME_ADMIN_QUEUE;\r
\r
    if (Index == 1) {\r
      QueueSize = NVME_CSQ_SIZE;\r
    } else {\r
      if (Private->Cap.Mqes > NVME_ASYNC_CSQ_SIZE) {\r
        QueueSize = NVME_ASYNC_CSQ_SIZE;\r
      } else {\r
        QueueSize = Private->Cap.Mqes;\r
      }\r
    }\r
\r
    CrIoSq.Qid   = Index;\r
    CrIoSq.Qsize = QueueSize;\r
    CrIoSq.Pc    = 1;\r
    CrIoSq.Cqid  = Index;\r
    CrIoSq.Qprio = 0;\r
    CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoSq, sizeof (NVME_ADMIN_CRIOSQ));\r
    CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;\r
\r
    Status = Private->Passthru.PassThru (\r
                                 &Private->Passthru,\r
                                 0,\r
                                 &CommandPacket,\r
                                 NULL\r
                                 );\r
    if (EFI_ERROR (Status)) {\r
      break;\r
    }\r
  }\r
\r
  Private->CreateIoQueue = FALSE;\r
\r
  return Status;\r
}\r
\r
/**\r
  Initialize the Nvm Express controller.\r
\r
  @param[in] Private                 The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.\r
\r
  @retval EFI_SUCCESS                The NVM Express Controller is initialized successfully.\r
  @retval Others                     A device error occurred while initializing the controller.\r
\r
**/\r
EFI_STATUS\r
NvmeControllerInit (\r
  IN NVME_CONTROLLER_PRIVATE_DATA    *Private\r
  )\r
{\r
  EFI_STATUS                      Status;\r
  EFI_PCI_IO_PROTOCOL             *PciIo;\r
  UINT64                          Supports;\r
  NVME_AQA                        Aqa;\r
  NVME_ASQ                        Asq;\r
  NVME_ACQ                        Acq;\r
  UINT8                           Sn[21];\r
  UINT8                           Mn[41];\r
  //\r
  // Save original PCI attributes and enable this controller.\r
  //\r
  PciIo  = Private->PciIo;\r
  Status = PciIo->Attributes (\r
                    PciIo,\r
                    EfiPciIoAttributeOperationGet,\r
                    0,\r
                    &Private->PciAttributes\r
                    );\r
\r
  if (EFI_ERROR (Status)) {\r
    return Status;\r
  }\r
\r
  Status = PciIo->Attributes (\r
                    PciIo,\r
                    EfiPciIoAttributeOperationSupported,\r
                    0,\r
                    &Supports\r
                    );\r
\r
  if (!EFI_ERROR (Status)) {\r
    Supports &= (UINT64)EFI_PCI_DEVICE_ENABLE;\r
    Status    = PciIo->Attributes (\r
                         PciIo,\r
                         EfiPciIoAttributeOperationEnable,\r
                         Supports,\r
                         NULL\r
                         );\r
  }\r
\r
  if (EFI_ERROR (Status)) {\r
    DEBUG ((EFI_D_INFO, "NvmeControllerInit: failed to enable controller\n"));\r
    return Status;\r
  }\r
\r
  //\r
  // Enable 64-bit DMA support in the PCI layer.\r
  //\r
  Status = PciIo->Attributes (\r
                    PciIo,\r
                    EfiPciIoAttributeOperationEnable,\r
                    EFI_PCI_IO_ATTRIBUTE_DUAL_ADDRESS_CYCLE,\r
                    NULL\r
                    );\r
  if (EFI_ERROR (Status)) {\r
    DEBUG ((EFI_D_WARN, "NvmeControllerInit: failed to enable 64-bit DMA (%r)\n", Status));\r
  }\r
\r
  //\r
  // Read the Controller Capabilities register and verify that the NVM command set is supported\r
  //\r
  Status = ReadNvmeControllerCapabilities (Private, &Private->Cap);\r
  if (EFI_ERROR (Status)) {\r
    return Status;\r
  }\r
\r
  if (Private->Cap.Css != 0x01) {\r
    DEBUG ((EFI_D_INFO, "NvmeControllerInit: the controller doesn't support NVMe command set\n"));\r
    return EFI_UNSUPPORTED;\r
  }\r
\r
  //\r
  // Currently the driver only supports 4k page size.\r
  //\r
  ASSERT ((Private->Cap.Mpsmin + 12) <= EFI_PAGE_SHIFT);\r
\r
  Private->Cid[0] = 0;\r
  Private->Cid[1] = 0;\r
  Private->Cid[2] = 0;\r
  Private->Pt[0]  = 0;\r
  Private->Pt[1]  = 0;\r
  Private->Pt[2]  = 0;\r
  Private->SqTdbl[0].Sqt = 0;\r
  Private->SqTdbl[1].Sqt = 0;\r
  Private->SqTdbl[2].Sqt = 0;\r
  Private->CqHdbl[0].Cqh = 0;\r
  Private->CqHdbl[1].Cqh = 0;\r
  Private->CqHdbl[2].Cqh = 0;\r
  Private->AsyncSqHead   = 0;\r
\r
  Status = NvmeDisableController (Private);\r
\r
  if (EFI_ERROR(Status)) {\r
    return Status;\r
  }\r
\r
  //\r
  // set number of entries admin submission & completion queues.\r
  //\r
  Aqa.Asqs  = NVME_ASQ_SIZE;\r
  Aqa.Rsvd1 = 0;\r
  Aqa.Acqs  = NVME_ACQ_SIZE;\r
  Aqa.Rsvd2 = 0;\r
\r
  //\r
  // Address of admin submission queue.\r
  //\r
  Asq = (UINT64)(UINTN)(Private->BufferPciAddr) & ~0xFFF;\r
\r
  //\r
  // Address of admin completion queue.\r
  //\r
  Acq = (UINT64)(UINTN)(Private->BufferPciAddr + EFI_PAGE_SIZE) & ~0xFFF;\r
\r
  //\r
  // Address of I/O submission & completion queue.\r
  //\r
  ZeroMem (Private->Buffer, EFI_PAGES_TO_SIZE (6));\r
  Private->SqBuffer[0]        = (NVME_SQ *)(UINTN)(Private->Buffer);\r
  Private->SqBufferPciAddr[0] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr);\r
  Private->CqBuffer[0]        = (NVME_CQ *)(UINTN)(Private->Buffer + 1 * EFI_PAGE_SIZE);\r
  Private->CqBufferPciAddr[0] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 1 * EFI_PAGE_SIZE);\r
  Private->SqBuffer[1]        = (NVME_SQ *)(UINTN)(Private->Buffer + 2 * EFI_PAGE_SIZE);\r
  Private->SqBufferPciAddr[1] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 2 * EFI_PAGE_SIZE);\r
  Private->CqBuffer[1]        = (NVME_CQ *)(UINTN)(Private->Buffer + 3 * EFI_PAGE_SIZE);\r
  Private->CqBufferPciAddr[1] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 3 * EFI_PAGE_SIZE);\r
  Private->SqBuffer[2]        = (NVME_SQ *)(UINTN)(Private->Buffer + 4 * EFI_PAGE_SIZE);\r
  Private->SqBufferPciAddr[2] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 4 * EFI_PAGE_SIZE);\r
  Private->CqBuffer[2]        = (NVME_CQ *)(UINTN)(Private->Buffer + 5 * EFI_PAGE_SIZE);\r
  Private->CqBufferPciAddr[2] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 5 * EFI_PAGE_SIZE);\r
\r
  DEBUG ((EFI_D_INFO, "Private->Buffer = [%016X]\n", (UINT64)(UINTN)Private->Buffer));\r
  DEBUG ((EFI_D_INFO, "Admin     Submission Queue size (Aqa.Asqs) = [%08X]\n", Aqa.Asqs));\r
  DEBUG ((EFI_D_INFO, "Admin     Completion Queue size (Aqa.Acqs) = [%08X]\n", Aqa.Acqs));\r
  DEBUG ((EFI_D_INFO, "Admin     Submission Queue (SqBuffer[0]) = [%016X]\n", Private->SqBuffer[0]));\r
  DEBUG ((EFI_D_INFO, "Admin     Completion Queue (CqBuffer[0]) = [%016X]\n", Private->CqBuffer[0]));\r
  DEBUG ((EFI_D_INFO, "Sync  I/O Submission Queue (SqBuffer[1]) = [%016X]\n", Private->SqBuffer[1]));\r
  DEBUG ((EFI_D_INFO, "Sync  I/O Completion Queue (CqBuffer[1]) = [%016X]\n", Private->CqBuffer[1]));\r
  DEBUG ((EFI_D_INFO, "Async I/O Submission Queue (SqBuffer[2]) = [%016X]\n", Private->SqBuffer[2]));\r
  DEBUG ((EFI_D_INFO, "Async I/O Completion Queue (CqBuffer[2]) = [%016X]\n", Private->CqBuffer[2]));\r
\r
  //\r
  // Program admin queue attributes.\r
  //\r
  Status = WriteNvmeAdminQueueAttributes (Private, &Aqa);\r
\r
  if (EFI_ERROR(Status)) {\r
    return Status;\r
  }\r
\r
  //\r
  // Program admin submission queue address.\r
  //\r
  Status = WriteNvmeAdminSubmissionQueueBaseAddress (Private, &Asq);\r
\r
  if (EFI_ERROR(Status)) {\r
    return Status;\r
  }\r
\r
  //\r
  // Program admin completion queue address.\r
  //\r
  Status = WriteNvmeAdminCompletionQueueBaseAddress (Private, &Acq);\r
\r
  if (EFI_ERROR(Status)) {\r
    return Status;\r
  }\r
\r
  Status = NvmeEnableController (Private);\r
  if (EFI_ERROR(Status)) {\r
    return Status;\r
  }\r
\r
  //\r
  // Allocate buffer for Identify Controller data\r
  //\r
  if (Private->ControllerData == NULL) {\r
    Private->ControllerData = (NVME_ADMIN_CONTROLLER_DATA *)AllocateZeroPool (sizeof(NVME_ADMIN_CONTROLLER_DATA));\r
\r
    if (Private->ControllerData == NULL) {\r
      return EFI_OUT_OF_RESOURCES;\r
    }\r
  }\r
\r
  //\r
  // Get current Identify Controller Data\r
  //\r
  Status = NvmeIdentifyController (Private, Private->ControllerData);\r
\r
  if (EFI_ERROR(Status)) {\r
    FreePool(Private->ControllerData);\r
    Private->ControllerData = NULL;\r
    return EFI_NOT_FOUND;\r
  }\r
\r
  //\r
  // Dump NvmExpress Identify Controller Data\r
  //\r
  CopyMem (Sn, Private->ControllerData->Sn, sizeof (Private->ControllerData->Sn));\r
  Sn[20] = 0;\r
  CopyMem (Mn, Private->ControllerData->Mn, sizeof (Private->ControllerData->Mn));\r
  Mn[40] = 0;\r
  DEBUG ((EFI_D_INFO, " == NVME IDENTIFY CONTROLLER DATA ==\n"));\r
  DEBUG ((EFI_D_INFO, "    PCI VID   : 0x%x\n", Private->ControllerData->Vid));\r
  DEBUG ((EFI_D_INFO, "    PCI SSVID : 0x%x\n", Private->ControllerData->Ssvid));\r
  DEBUG ((EFI_D_INFO, "    SN        : %a\n",   Sn));\r
  DEBUG ((EFI_D_INFO, "    MN        : %a\n",   Mn));\r
  DEBUG ((EFI_D_INFO, "    FR        : 0x%x\n", *((UINT64*)Private->ControllerData->Fr)));\r
  DEBUG ((EFI_D_INFO, "    RAB       : 0x%x\n", Private->ControllerData->Rab));\r
  DEBUG ((EFI_D_INFO, "    IEEE      : 0x%x\n", *(UINT32*)Private->ControllerData->Ieee_oui));\r
  DEBUG ((EFI_D_INFO, "    AERL      : 0x%x\n", Private->ControllerData->Aerl));\r
  DEBUG ((EFI_D_INFO, "    SQES      : 0x%x\n", Private->ControllerData->Sqes));\r
  DEBUG ((EFI_D_INFO, "    CQES      : 0x%x\n", Private->ControllerData->Cqes));\r
  DEBUG ((EFI_D_INFO, "    NN        : 0x%x\n", Private->ControllerData->Nn));\r
\r
  //\r
  // Create two I/O completion queues.\r
  // One for blocking I/O, one for non-blocking I/O.\r
  //\r
  Status = NvmeCreateIoCompletionQueue (Private);\r
  if (EFI_ERROR(Status)) {\r
   return Status;\r
  }\r
\r
  //\r
  // Create two I/O Submission queues.\r
  // One for blocking I/O, one for non-blocking I/O.\r
  //\r
  Status = NvmeCreateIoSubmissionQueue (Private);\r
\r
  return Status;\r
}\r
\r
/**\r
 This routine is called to properly shutdown the Nvm Express controller per NVMe spec.\r
\r
  @param[in]  ResetType         The type of reset to perform.\r
  @param[in]  ResetStatus       The status code for the reset.\r
  @param[in]  DataSize          The size, in bytes, of ResetData.\r
  @param[in]  ResetData         For a ResetType of EfiResetCold, EfiResetWarm, or\r
                                EfiResetShutdown the data buffer starts with a Null-terminated\r
                                string, optionally followed by additional binary data.\r
                                The string is a description that the caller may use to further\r
                                indicate the reason for the system reset. ResetData is only\r
                                valid if ResetStatus is something other than EFI_SUCCESS\r
                                unless the ResetType is EfiResetPlatformSpecific\r
                                where a minimum amount of ResetData is always required.\r
                                For a ResetType of EfiResetPlatformSpecific the data buffer\r
                                also starts with a Null-terminated string that is followed\r
                                by an EFI_GUID that describes the specific type of reset to perform.\r
**/\r
VOID\r
EFIAPI\r
NvmeShutdownAllControllers (\r
  IN EFI_RESET_TYPE           ResetType,\r
  IN EFI_STATUS               ResetStatus,\r
  IN UINTN                    DataSize,\r
  IN VOID                     *ResetData OPTIONAL\r
  )\r
{\r
  EFI_STATUS                          Status;\r
  EFI_HANDLE                          *Handles;\r
  UINTN                               HandleCount;\r
  UINTN                               HandleIndex;\r
  EFI_OPEN_PROTOCOL_INFORMATION_ENTRY *OpenInfos;\r
  UINTN                               OpenInfoCount;\r
  UINTN                               OpenInfoIndex;\r
  EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL  *NvmePassThru;\r
  NVME_CC                             Cc;\r
  NVME_CSTS                           Csts;\r
  UINTN                               Index;\r
  NVME_CONTROLLER_PRIVATE_DATA        *Private;\r
\r
  Status = gBS->LocateHandleBuffer (\r
                  ByProtocol,\r
                  &gEfiPciIoProtocolGuid,\r
                  NULL,\r
                  &HandleCount,\r
                  &Handles\r
                  );\r
  if (EFI_ERROR (Status)) {\r
    HandleCount = 0;\r
  }\r
\r
  for (HandleIndex = 0; HandleIndex < HandleCount; HandleIndex++) {\r
    Status = gBS->OpenProtocolInformation (\r
                    Handles[HandleIndex],\r
                    &gEfiPciIoProtocolGuid,\r
                    &OpenInfos,\r
                    &OpenInfoCount\r
                    );\r
    if (EFI_ERROR (Status)) {\r
      continue;\r
    }\r
\r
    for (OpenInfoIndex = 0; OpenInfoIndex < OpenInfoCount; OpenInfoIndex++) {\r
      //\r
      // Find all the NVME controller managed by this driver.\r
      // gImageHandle equals to DriverBinding handle for this driver.\r
      //\r
      if (((OpenInfos[OpenInfoIndex].Attributes & EFI_OPEN_PROTOCOL_BY_DRIVER) != 0) &&\r
          (OpenInfos[OpenInfoIndex].AgentHandle == gImageHandle)) {\r
        Status = gBS->OpenProtocol (\r
                        OpenInfos[OpenInfoIndex].ControllerHandle,\r
                        &gEfiNvmExpressPassThruProtocolGuid,\r
                        (VOID **) &NvmePassThru,\r
                        NULL,\r
                        NULL,\r
                        EFI_OPEN_PROTOCOL_GET_PROTOCOL\r
                        );\r
        if (EFI_ERROR (Status)) {\r
          continue;\r
        }\r
        Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (NvmePassThru);\r
\r
        //\r
        // Read Controller Configuration Register.\r
        //\r
        Status = ReadNvmeControllerConfiguration (Private, &Cc);\r
        if (EFI_ERROR(Status)) {\r
          continue;\r
        }\r
        //\r
        // The host should set the Shutdown Notification (CC.SHN) field to 01b\r
        // to indicate a normal shutdown operation.\r
        //\r
        Cc.Shn = NVME_CC_SHN_NORMAL_SHUTDOWN;\r
        Status = WriteNvmeControllerConfiguration (Private, &Cc);\r
        if (EFI_ERROR(Status)) {\r
          continue;\r
        }\r
\r
        //\r
        // The controller indicates when shutdown processing is completed by updating the\r
        // Shutdown Status (CSTS.SHST) field to 10b.\r
        // Wait up to 45 seconds (break down to 4500 x 10ms) for the shutdown to complete.\r
        //\r
        for (Index = 0; Index < NVME_SHUTDOWN_PROCESS_TIMEOUT * 100; Index++) {\r
          Status = ReadNvmeControllerStatus (Private, &Csts);\r
          if (!EFI_ERROR(Status) && (Csts.Shst == NVME_CSTS_SHST_SHUTDOWN_COMPLETED)) {\r
            DEBUG((DEBUG_INFO, "NvmeShutdownController: shutdown processing is completed after %dms.\n", Index * 10));\r
            break;\r
          }\r
          //\r
          // Stall for 10ms\r
          //\r
          gBS->Stall (10 * 1000);\r
        }\r
\r
        if (Index == NVME_SHUTDOWN_PROCESS_TIMEOUT * 100) {\r
          DEBUG((DEBUG_ERROR, "NvmeShutdownController: shutdown processing is timed out\n"));\r
        }\r
      }\r
    }\r
  }\r
}\r
\r
/**\r
  Register the shutdown notification through the ResetNotification protocol.\r
\r
  Register the shutdown notification when mNvmeControllerNumber increased from 0 to 1.\r
**/\r
VOID\r
NvmeRegisterShutdownNotification (\r
  VOID\r
  )\r
{\r
  EFI_STATUS                      Status;\r
  EFI_RESET_NOTIFICATION_PROTOCOL *ResetNotify;\r
\r
  mNvmeControllerNumber++;\r
  if (mNvmeControllerNumber == 1) {\r
    Status = gBS->LocateProtocol (&gEfiResetNotificationProtocolGuid, NULL, (VOID **) &ResetNotify);\r
    if (!EFI_ERROR (Status)) {\r
      Status = ResetNotify->RegisterResetNotify (ResetNotify, NvmeShutdownAllControllers);\r
      ASSERT_EFI_ERROR (Status);\r
    } else {\r
      DEBUG ((DEBUG_WARN, "NVME: ResetNotification absent! Shutdown notification cannot be performed!\n"));\r
    }\r
  }\r
}\r
\r
/**\r
  Unregister the shutdown notification through the ResetNotification protocol.\r
\r
  Unregister the shutdown notification when mNvmeControllerNumber decreased from 1 to 0.\r
**/\r
VOID\r
NvmeUnregisterShutdownNotification (\r
  VOID\r
  )\r
{\r
  EFI_STATUS                      Status;\r
  EFI_RESET_NOTIFICATION_PROTOCOL *ResetNotify;\r
\r
  mNvmeControllerNumber--;\r
  if (mNvmeControllerNumber == 0) {\r
    Status = gBS->LocateProtocol (&gEfiResetNotificationProtocolGuid, NULL, (VOID **) &ResetNotify);\r
    if (!EFI_ERROR (Status)) {\r
      Status = ResetNotify->UnregisterResetNotify (ResetNotify, NvmeShutdownAllControllers);\r
      ASSERT_EFI_ERROR (Status);\r
    }\r
  }\r
}\r