MdeModulePkg/Bus/Pci/NvmExpressPei/NvmExpressPeiHci.c

   1 /** @file
   2   The NvmExpressPei driver is used to manage non-volatile memory subsystem
   3   which follows NVM Express specification at PEI phase.
   4
   5   Copyright (c) 2018 - 2019, Intel Corporation. All rights reserved.<BR>
   6
   7   SPDX-License-Identifier: BSD-2-Clause-Patent
   8
   9 **/
  10
  11 #include "NvmExpressPei.h"
  12
  13 /**
  14   Transfer MMIO Data to memory.
  15
  16   @param[in,out] MemBuffer    Destination: Memory address.
  17   @param[in] MmioAddr         Source: MMIO address.
  18   @param[in] Size             Size for read.
  19
  20   @retval EFI_SUCCESS         MMIO read sucessfully.
  21
  22 **/
  23 EFI_STATUS
  24 NvmeMmioRead (
  25   IN OUT VOID   *MemBuffer,
  26   IN     UINTN  MmioAddr,
  27   IN     UINTN  Size
  28   )
  29 {
  30   UINTN  Offset;
  31   UINT8  Data;
  32   UINT8  *Ptr;
  33
  34   // priority has adjusted
  35   switch (Size) {
  36     case 4:
  37       *((UINT32 *)MemBuffer) = MmioRead32 (MmioAddr);
  38       break;
  39
  40     case 8:
  41       *((UINT64 *)MemBuffer) = MmioRead64 (MmioAddr);
  42       break;
  43
  44     case 2:
  45       *((UINT16 *)MemBuffer) = MmioRead16 (MmioAddr);
  46       break;
  47
  48     case 1:
  49       *((UINT8 *)MemBuffer) = MmioRead8 (MmioAddr);
  50       break;
  51
  52     default:
  53       Ptr = (UINT8 *)MemBuffer;
  54       for (Offset = 0; Offset < Size; Offset += 1) {
  55         Data        = MmioRead8 (MmioAddr + Offset);
  56         Ptr[Offset] = Data;
  57       }
  58
  59       break;
  60   }
  61
  62   return EFI_SUCCESS;
  63 }
  64
  65 /**
  66   Transfer memory data to MMIO.
  67
  68   @param[in,out] MmioAddr    Destination: MMIO address.
  69   @param[in] MemBuffer       Source: Memory address.
  70   @param[in] Size            Size for write.
  71
  72   @retval EFI_SUCCESS        MMIO write sucessfully.
  73
  74 **/
  75 EFI_STATUS
  76 NvmeMmioWrite (
  77   IN OUT UINTN  MmioAddr,
  78   IN     VOID   *MemBuffer,
  79   IN     UINTN  Size
  80   )
  81 {
  82   UINTN  Offset;
  83   UINT8  Data;
  84   UINT8  *Ptr;
  85
  86   // priority has adjusted
  87   switch (Size) {
  88     case 4:
  89       MmioWrite32 (MmioAddr, *((UINT32 *)MemBuffer));
  90       break;
  91
  92     case 8:
  93       MmioWrite64 (MmioAddr, *((UINT64 *)MemBuffer));
  94       break;
  95
  96     case 2:
  97       MmioWrite16 (MmioAddr, *((UINT16 *)MemBuffer));
  98       break;
  99
 100     case 1:
 101       MmioWrite8 (MmioAddr, *((UINT8 *)MemBuffer));
 102       break;
 103
 104     default:
 105       Ptr = (UINT8 *)MemBuffer;
 106       for (Offset = 0; Offset < Size; Offset += 1) {
 107         Data = Ptr[Offset];
 108         MmioWrite8 (MmioAddr + Offset, Data);
 109       }
 110
 111       break;
 112   }
 113
 114   return EFI_SUCCESS;
 115 }
 116
 117 /**
 118   Get the page offset for specific NVME based memory.
 119
 120   @param[in] BaseMemIndex    The Index of BaseMem (0-based).
 121
 122   @retval - The page count for specific BaseMem Index
 123
 124 **/
 125 UINT32
 126 NvmeBaseMemPageOffset (
 127   IN UINTN  BaseMemIndex
 128   )
 129 {
 130   UINT32  Pages;
 131   UINTN   Index;
 132   UINT32  PageSizeList[5];
 133
 134   PageSizeList[0] = 1;             /* ASQ */
 135   PageSizeList[1] = 1;             /* ACQ */
 136   PageSizeList[2] = 1;             /* SQs */
 137   PageSizeList[3] = 1;             /* CQs */
 138   PageSizeList[4] = NVME_PRP_SIZE; /* PRPs */
 139
 140   if (BaseMemIndex > MAX_BASEMEM_COUNT) {
 141     DEBUG ((DEBUG_ERROR, "%a: The input BaseMem index is invalid.\n", __FUNCTION__));
 142     ASSERT (FALSE);
 143     return 0;
 144   }
 145
 146   Pages = 0;
 147   for (Index = 0; Index < BaseMemIndex; Index++) {
 148     Pages += PageSizeList[Index];
 149   }
 150
 151   return Pages;
 152 }
 153
 154 /**
 155   Wait for NVME controller status to be ready or not.
 156
 157   @param[in] Private      The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 158   @param[in] WaitReady    Flag for waitting status ready or not.
 159
 160   @return EFI_SUCCESS     Successfully to wait specific status.
 161   @return others          Fail to wait for specific controller status.
 162
 163 **/
 164 EFI_STATUS
 165 NvmeWaitController (
 166   IN PEI_NVME_CONTROLLER_PRIVATE_DATA  *Private,
 167   IN BOOLEAN                           WaitReady
 168   )
 169 {
 170   NVME_CSTS   Csts;
 171   EFI_STATUS  Status;
 172   UINT32      Index;
 173   UINT8       Timeout;
 174
 175   //
 176   // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to set after
 177   // Cc.Enable. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To.
 178   //
 179   if (Private->Cap.To == 0) {
 180     Timeout = 1;
 181   } else {
 182     Timeout = Private->Cap.To;
 183   }
 184
 185   Status = EFI_SUCCESS;
 186   for (Index = (Timeout * 500); Index != 0; --Index) {
 187     MicroSecondDelay (1000);
 188
 189     //
 190     // Check if the controller is initialized
 191     //
 192     Status = NVME_GET_CSTS (Private, &Csts);
 193     if (EFI_ERROR (Status)) {
 194       DEBUG ((DEBUG_ERROR, "%a: NVME_GET_CSTS fail, Status - %r\n", __FUNCTION__, Status));
 195       return Status;
 196     }
 197
 198     if ((BOOLEAN)Csts.Rdy == WaitReady) {
 199       break;
 200     }
 201   }
 202
 203   if (Index == 0) {
 204     Status = EFI_TIMEOUT;
 205   }
 206
 207   return Status;
 208 }
 209
 210 /**
 211   Disable the Nvm Express controller.
 212
 213   @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 214
 215   @return EFI_SUCCESS    Successfully disable the controller.
 216   @return others         Fail to disable the controller.
 217
 218 **/
 219 EFI_STATUS
 220 NvmeDisableController (
 221   IN PEI_NVME_CONTROLLER_PRIVATE_DATA  *Private
 222   )
 223 {
 224   NVME_CC     Cc;
 225   NVME_CSTS   Csts;
 226   EFI_STATUS  Status;
 227
 228   Status = NVME_GET_CSTS (Private, &Csts);
 229
 230   //
 231   // Read Controller Configuration Register.
 232   //
 233   Status = NVME_GET_CC (Private, &Cc);
 234   if (EFI_ERROR (Status)) {
 235     DEBUG ((DEBUG_ERROR, "%a: NVME_GET_CC fail, Status - %r\n", __FUNCTION__, Status));
 236     goto ErrorExit;
 237   }
 238
 239   if (Cc.En == 1) {
 240     Cc.En = 0;
 241     //
 242     // Disable the controller.
 243     //
 244     Status = NVME_SET_CC (Private, &Cc);
 245     if (EFI_ERROR (Status)) {
 246       DEBUG ((DEBUG_ERROR, "%a: NVME_SET_CC fail, Status - %r\n", __FUNCTION__, Status));
 247       goto ErrorExit;
 248     }
 249   }
 250
 251   Status = NvmeWaitController (Private, FALSE);
 252   if (EFI_ERROR (Status)) {
 253     DEBUG ((DEBUG_ERROR, "%a: NvmeWaitController fail, Status - %r\n", __FUNCTION__, Status));
 254     goto ErrorExit;
 255   }
 256
 257   return EFI_SUCCESS;
 258
 259 ErrorExit:
 260   DEBUG ((DEBUG_ERROR, "%a fail, Status - %r\n", __FUNCTION__, Status));
 261   return Status;
 262 }
 263
 264 /**
 265   Enable the Nvm Express controller.
 266
 267   @param[in] Private    The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 268
 269   @return EFI_SUCCESS         Successfully enable the controller.
 270   @return EFI_DEVICE_ERROR    Fail to enable the controller.
 271   @return EFI_TIMEOUT         Fail to enable the controller in given time slot.
 272
 273 **/
 274 EFI_STATUS
 275 NvmeEnableController (
 276   IN PEI_NVME_CONTROLLER_PRIVATE_DATA  *Private
 277   )
 278 {
 279   NVME_CC     Cc;
 280   EFI_STATUS  Status;
 281
 282   //
 283   // Enable the controller
 284   // CC.AMS, CC.MPS and CC.CSS are all set to 0
 285   //
 286   ZeroMem (&Cc, sizeof (NVME_CC));
 287   Cc.En     = 1;
 288   Cc.Iosqes = 6;
 289   Cc.Iocqes = 4;
 290   Status    = NVME_SET_CC (Private, &Cc);
 291   if (EFI_ERROR (Status)) {
 292     DEBUG ((DEBUG_ERROR, "%a: NVME_SET_CC fail, Status - %r\n", __FUNCTION__, Status));
 293     goto ErrorExit;
 294   }
 295
 296   Status = NvmeWaitController (Private, TRUE);
 297   if (EFI_ERROR (Status)) {
 298     DEBUG ((DEBUG_ERROR, "%a: NvmeWaitController fail, Status - %r\n", __FUNCTION__, Status));
 299     goto ErrorExit;
 300   }
 301
 302   return EFI_SUCCESS;
 303
 304 ErrorExit:
 305   DEBUG ((DEBUG_ERROR, "%a fail, Status: %r\n", __FUNCTION__, Status));
 306   return Status;
 307 }
 308
 309 /**
 310   Get the Identify Controller data.
 311
 312   @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 313   @param[in] Buffer      The Buffer used to store the Identify Controller data.
 314
 315   @return EFI_SUCCESS    Successfully get the Identify Controller data.
 316   @return others         Fail to get the Identify Controller data.
 317
 318 **/
 319 EFI_STATUS
 320 NvmeIdentifyController (
 321   IN PEI_NVME_CONTROLLER_PRIVATE_DATA  *Private,
 322   IN VOID                              *Buffer
 323   )
 324 {
 325   EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET  CommandPacket;
 326   EFI_NVM_EXPRESS_COMMAND                   Command;
 327   EFI_NVM_EXPRESS_COMPLETION                Completion;
 328   EFI_STATUS                                Status;
 329
 330   ZeroMem (&CommandPacket, sizeof (EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
 331   ZeroMem (&Command, sizeof (EFI_NVM_EXPRESS_COMMAND));
 332   ZeroMem (&Completion, sizeof (EFI_NVM_EXPRESS_COMPLETION));
 333
 334   Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD;
 335   //
 336   // According to Nvm Express 1.1 spec Figure 38, When not used, the field shall be cleared to 0h.
 337   // For the Identify command, the Namespace Identifier is only used for the Namespace Data structure.
 338   //
 339   Command.Nsid = 0;
 340
 341   CommandPacket.NvmeCmd        = &Command;
 342   CommandPacket.NvmeCompletion = &Completion;
 343   CommandPacket.TransferBuffer = Buffer;
 344   CommandPacket.TransferLength = sizeof (NVME_ADMIN_CONTROLLER_DATA);
 345   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
 346   CommandPacket.QueueType      = NVME_ADMIN_QUEUE;
 347   //
 348   // Set bit 0 (Cns bit) to 1 to identify the controller
 349   //
 350   CommandPacket.NvmeCmd->Cdw10 = 1;
 351   CommandPacket.NvmeCmd->Flags = CDW10_VALID;
 352
 353   Status = NvmePassThruExecute (
 354              Private,
 355              NVME_CONTROLLER_NSID,
 356              &CommandPacket
 357              );
 358   return Status;
 359 }
 360
 361 /**
 362   Get specified identify namespace data.
 363
 364   @param[in] Private        The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 365   @param[in] NamespaceId    The specified namespace identifier.
 366   @param[in] Buffer         The buffer used to store the identify namespace data.
 367
 368   @return EFI_SUCCESS         Successfully get the identify namespace data.
 369   @return EFI_DEVICE_ERROR    Fail to get the identify namespace data.
 370
 371 **/
 372 EFI_STATUS
 373 NvmeIdentifyNamespace (
 374   IN PEI_NVME_CONTROLLER_PRIVATE_DATA  *Private,
 375   IN UINT32                            NamespaceId,
 376   IN VOID                              *Buffer
 377   )
 378 {
 379   EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET  CommandPacket;
 380   EFI_NVM_EXPRESS_COMMAND                   Command;
 381   EFI_NVM_EXPRESS_COMPLETION                Completion;
 382   EFI_STATUS                                Status;
 383
 384   ZeroMem (&CommandPacket, sizeof (EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
 385   ZeroMem (&Command, sizeof (EFI_NVM_EXPRESS_COMMAND));
 386   ZeroMem (&Completion, sizeof (EFI_NVM_EXPRESS_COMPLETION));
 387
 388   Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD;
 389   Command.Nsid        = NamespaceId;
 390
 391   CommandPacket.NvmeCmd        = &Command;
 392   CommandPacket.NvmeCompletion = &Completion;
 393   CommandPacket.TransferBuffer = Buffer;
 394   CommandPacket.TransferLength = sizeof (NVME_ADMIN_NAMESPACE_DATA);
 395   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
 396   CommandPacket.QueueType      = NVME_ADMIN_QUEUE;
 397   //
 398   // Set bit 0 (Cns bit) to 1 to identify a namespace
 399   //
 400   CommandPacket.NvmeCmd->Cdw10 = 0;
 401   CommandPacket.NvmeCmd->Flags = CDW10_VALID;
 402
 403   Status = NvmePassThruExecute (
 404              Private,
 405              NamespaceId,
 406              &CommandPacket
 407              );
 408   return Status;
 409 }
 410
 411 /**
 412   Dump the Identify Controller data.
 413
 414   @param[in] ControllerData    The pointer to the NVME_ADMIN_CONTROLLER_DATA data structure.
 415
 416 **/
 417 VOID
 418 NvmeDumpControllerData (
 419   IN NVME_ADMIN_CONTROLLER_DATA  *ControllerData
 420   )
 421 {
 422   UINT8  Sn[21];
 423   UINT8  Mn[41];
 424
 425   CopyMem (Sn, ControllerData->Sn, sizeof (ControllerData->Sn));
 426   Sn[20] = 0;
 427   CopyMem (Mn, ControllerData->Mn, sizeof (ControllerData->Mn));
 428   Mn[40] = 0;
 429
 430   DEBUG ((DEBUG_INFO, " == NVME IDENTIFY CONTROLLER DATA ==\n"));
 431   DEBUG ((DEBUG_INFO, "    PCI VID   : 0x%x\n", ControllerData->Vid));
 432   DEBUG ((DEBUG_INFO, "    PCI SSVID : 0x%x\n", ControllerData->Ssvid));
 433   DEBUG ((DEBUG_INFO, "    SN        : %a\n", Sn));
 434   DEBUG ((DEBUG_INFO, "    MN        : %a\n", Mn));
 435   DEBUG ((DEBUG_INFO, "    FR        : 0x%lx\n", *((UINT64 *)ControllerData->Fr)));
 436   DEBUG ((DEBUG_INFO, "    RAB       : 0x%x\n", ControllerData->Rab));
 437   DEBUG ((DEBUG_INFO, "    IEEE      : 0x%x\n", *(UINT32 *)ControllerData->Ieee_oui));
 438   DEBUG ((DEBUG_INFO, "    AERL      : 0x%x\n", ControllerData->Aerl));
 439   DEBUG ((DEBUG_INFO, "    SQES      : 0x%x\n", ControllerData->Sqes));
 440   DEBUG ((DEBUG_INFO, "    CQES      : 0x%x\n", ControllerData->Cqes));
 441   DEBUG ((DEBUG_INFO, "    NN        : 0x%x\n", ControllerData->Nn));
 442   return;
 443 }
 444
 445 /**
 446   Create IO completion queue.
 447
 448   @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 449
 450   @return EFI_SUCCESS    Successfully create io completion queue.
 451   @return others         Fail to create io completion queue.
 452
 453 **/
 454 EFI_STATUS
 455 NvmeCreateIoCompletionQueue (
 456   IN PEI_NVME_CONTROLLER_PRIVATE_DATA  *Private
 457   )
 458 {
 459   EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET  CommandPacket;
 460   EFI_NVM_EXPRESS_COMMAND                   Command;
 461   EFI_NVM_EXPRESS_COMPLETION                Completion;
 462   EFI_STATUS                                Status;
 463   NVME_ADMIN_CRIOCQ                         CrIoCq;
 464
 465   ZeroMem (&CommandPacket, sizeof (EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
 466   ZeroMem (&Command, sizeof (EFI_NVM_EXPRESS_COMMAND));
 467   ZeroMem (&Completion, sizeof (EFI_NVM_EXPRESS_COMPLETION));
 468   ZeroMem (&CrIoCq, sizeof (NVME_ADMIN_CRIOCQ));
 469
 470   CommandPacket.NvmeCmd        = &Command;
 471   CommandPacket.NvmeCompletion = &Completion;
 472
 473   Command.Cdw0.Opcode          = NVME_ADMIN_CRIOCQ_CMD;
 474   CommandPacket.TransferBuffer = Private->CqBuffer[NVME_IO_QUEUE];
 475   CommandPacket.TransferLength = EFI_PAGE_SIZE;
 476   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
 477   CommandPacket.QueueType      = NVME_ADMIN_QUEUE;
 478
 479   CrIoCq.Qid   = NVME_IO_QUEUE;
 480   CrIoCq.Qsize = NVME_CCQ_SIZE;
 481   CrIoCq.Pc    = 1;
 482   CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoCq, sizeof (NVME_ADMIN_CRIOCQ));
 483   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
 484
 485   Status = NvmePassThruExecute (
 486              Private,
 487              NVME_CONTROLLER_NSID,
 488              &CommandPacket
 489              );
 490   return Status;
 491 }
 492
 493 /**
 494   Create IO submission queue.
 495
 496   @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 497
 498   @return EFI_SUCCESS    Successfully create io submission queue.
 499   @return others         Fail to create io submission queue.
 500
 501 **/
 502 EFI_STATUS
 503 NvmeCreateIoSubmissionQueue (
 504   IN PEI_NVME_CONTROLLER_PRIVATE_DATA  *Private
 505   )
 506 {
 507   EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET  CommandPacket;
 508   EFI_NVM_EXPRESS_COMMAND                   Command;
 509   EFI_NVM_EXPRESS_COMPLETION                Completion;
 510   EFI_STATUS                                Status;
 511   NVME_ADMIN_CRIOSQ                         CrIoSq;
 512
 513   ZeroMem (&CommandPacket, sizeof (EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
 514   ZeroMem (&Command, sizeof (EFI_NVM_EXPRESS_COMMAND));
 515   ZeroMem (&Completion, sizeof (EFI_NVM_EXPRESS_COMPLETION));
 516   ZeroMem (&CrIoSq, sizeof (NVME_ADMIN_CRIOSQ));
 517
 518   CommandPacket.NvmeCmd        = &Command;
 519   CommandPacket.NvmeCompletion = &Completion;
 520
 521   Command.Cdw0.Opcode          = NVME_ADMIN_CRIOSQ_CMD;
 522   CommandPacket.TransferBuffer = Private->SqBuffer[NVME_IO_QUEUE];
 523   CommandPacket.TransferLength = EFI_PAGE_SIZE;
 524   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
 525   CommandPacket.QueueType      = NVME_ADMIN_QUEUE;
 526
 527   CrIoSq.Qid   = NVME_IO_QUEUE;
 528   CrIoSq.Qsize = NVME_CSQ_SIZE;
 529   CrIoSq.Pc    = 1;
 530   CrIoSq.Cqid  = NVME_IO_QUEUE;
 531   CrIoSq.Qprio = 0;
 532   CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoSq, sizeof (NVME_ADMIN_CRIOSQ));
 533   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
 534
 535   Status = NvmePassThruExecute (
 536              Private,
 537              NVME_CONTROLLER_NSID,
 538              &CommandPacket
 539              );
 540   return Status;
 541 }
 542
 543 /**
 544   Initialize the Nvm Express controller.
 545
 546   @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 547
 548   @retval EFI_SUCCESS    The NVM Express Controller is initialized successfully.
 549   @retval Others         A device error occurred while initializing the controller.
 550
 551 **/
 552 EFI_STATUS
 553 NvmeControllerInit (
 554   IN PEI_NVME_CONTROLLER_PRIVATE_DATA  *Private
 555   )
 556 {
 557   EFI_STATUS  Status;
 558   UINTN       Index;
 559   NVME_AQA    Aqa;
 560   NVME_ASQ    Asq;
 561   NVME_ACQ    Acq;
 562   NVME_VER    Ver;
 563
 564   //
 565   // Dump the NVME controller implementation version
 566   //
 567   NVME_GET_VER (Private, &Ver);
 568   DEBUG ((DEBUG_INFO, "NVME controller implementation version: %d.%d\n", Ver.Mjr, Ver.Mnr));
 569
 570   //
 571   // Read the controller Capabilities register and verify that the NVM command set is supported
 572   //
 573   NVME_GET_CAP (Private, &Private->Cap);
 574   if ((Private->Cap.Css & BIT0) == 0) {
 575     DEBUG ((DEBUG_ERROR, "%a: The NVME controller doesn't support NVMe command set.\n", __FUNCTION__));
 576     return EFI_UNSUPPORTED;
 577   }
 578
 579   //
 580   // Currently, the driver only supports 4k page size
 581   //
 582   if ((Private->Cap.Mpsmin + 12) > EFI_PAGE_SHIFT) {
 583     DEBUG ((DEBUG_ERROR, "%a: The driver doesn't support page size other than 4K.\n", __FUNCTION__));
 584     ASSERT (FALSE);
 585     return EFI_UNSUPPORTED;
 586   }
 587
 588   for (Index = 0; Index < NVME_MAX_QUEUES; Index++) {
 589     Private->Pt[Index]  = 0;
 590     Private->Cid[Index] = 0;
 591     ZeroMem ((VOID *)(UINTN)(&Private->SqTdbl[Index]), sizeof (NVME_SQTDBL));
 592     ZeroMem ((VOID *)(UINTN)(&Private->CqHdbl[Index]), sizeof (NVME_CQHDBL));
 593   }
 594
 595   ZeroMem (Private->Buffer, EFI_PAGE_SIZE * NVME_MEM_MAX_PAGES);
 596
 597   //
 598   // Disable the NVME controller first
 599   //
 600   Status = NvmeDisableController (Private);
 601   if (EFI_ERROR (Status)) {
 602     DEBUG ((DEBUG_ERROR, "%a: NvmeDisableController fail, Status - %r\n", __FUNCTION__, Status));
 603     return Status;
 604   }
 605
 606   //
 607   // Set the number of entries in admin submission & completion queues
 608   //
 609   Aqa.Asqs  = NVME_ASQ_SIZE;
 610   Aqa.Rsvd1 = 0;
 611   Aqa.Acqs  = NVME_ACQ_SIZE;
 612   Aqa.Rsvd2 = 0;
 613
 614   //
 615   // Address of admin submission & completion queues
 616   //
 617   Asq = (UINT64)(UINTN)(NVME_ASQ_BASE (Private) & ~0xFFF);
 618   Acq = (UINT64)(UINTN)(NVME_ACQ_BASE (Private) & ~0xFFF);
 619
 620   //
 621   // Address of I/O submission & completion queues
 622   //
 623   Private->SqBuffer[0] = (NVME_SQ *)(UINTN)NVME_ASQ_BASE (Private);    // NVME_ADMIN_QUEUE
 624   Private->CqBuffer[0] = (NVME_CQ *)(UINTN)NVME_ACQ_BASE (Private);    // NVME_ADMIN_QUEUE
 625   Private->SqBuffer[1] = (NVME_SQ *)(UINTN)NVME_SQ_BASE (Private, 0);  // NVME_IO_QUEUE
 626   Private->CqBuffer[1] = (NVME_CQ *)(UINTN)NVME_CQ_BASE (Private, 0);  // NVME_IO_QUEUE
 627   DEBUG ((DEBUG_INFO, "Admin Submission Queue Size (Aqa.Asqs) = [%08X]\n", Aqa.Asqs));
 628   DEBUG ((DEBUG_INFO, "Admin Completion Queue Size (Aqa.Acqs) = [%08X]\n", Aqa.Acqs));
 629   DEBUG ((DEBUG_INFO, "Admin Submission Queue (SqBuffer[0]) =   [%08X]\n", Private->SqBuffer[0]));
 630   DEBUG ((DEBUG_INFO, "Admin Completion Queue (CqBuffer[0]) =   [%08X]\n", Private->CqBuffer[0]));
 631   DEBUG ((DEBUG_INFO, "I/O   Submission Queue (SqBuffer[1]) =   [%08X]\n", Private->SqBuffer[1]));
 632   DEBUG ((DEBUG_INFO, "I/O   Completion Queue (CqBuffer[1]) =   [%08X]\n", Private->CqBuffer[1]));
 633
 634   //
 635   // Program admin queue attributes
 636   //
 637   NVME_SET_AQA (Private, &Aqa);
 638
 639   //
 640   // Program admin submission & completion queues address
 641   //
 642   NVME_SET_ASQ (Private, &Asq);
 643   NVME_SET_ACQ (Private, &Acq);
 644
 645   //
 646   // Enable the NVME controller
 647   //
 648   Status = NvmeEnableController (Private);
 649   if (EFI_ERROR (Status)) {
 650     DEBUG ((DEBUG_ERROR, "%a: NvmeEnableController fail, Status - %r\n", __FUNCTION__, Status));
 651     return Status;
 652   }
 653
 654   //
 655   // Get the Identify Controller data
 656   //
 657   if (Private->ControllerData == NULL) {
 658     Private->ControllerData = (NVME_ADMIN_CONTROLLER_DATA *)AllocateZeroPool (sizeof (NVME_ADMIN_CONTROLLER_DATA));
 659     if (Private->ControllerData == NULL) {
 660       return EFI_OUT_OF_RESOURCES;
 661     }
 662   }
 663
 664   Status = NvmeIdentifyController (Private, Private->ControllerData);
 665   if (EFI_ERROR (Status)) {
 666     DEBUG ((DEBUG_ERROR, "%a: NvmeIdentifyController fail, Status - %r\n", __FUNCTION__, Status));
 667     return Status;
 668   }
 669
 670   NvmeDumpControllerData (Private->ControllerData);
 671
 672   //
 673   // Check the namespace number for storing the namespaces information
 674   //
 675   if (Private->ControllerData->Nn > MAX_UINT32 / sizeof (PEI_NVME_NAMESPACE_INFO)) {
 676     DEBUG ((
 677       DEBUG_ERROR,
 678       "%a: Number of Namespaces field in Identify Controller data not supported by the driver.\n",
 679       __FUNCTION__
 680       ));
 681     return EFI_UNSUPPORTED;
 682   }
 683
 684   //
 685   // Create one I/O completion queue and one I/O submission queue
 686   //
 687   Status = NvmeCreateIoCompletionQueue (Private);
 688   if (EFI_ERROR (Status)) {
 689     DEBUG ((DEBUG_ERROR, "%a: Create IO completion queue fail, Status - %r\n", __FUNCTION__, Status));
 690     return Status;
 691   }
 692
 693   Status = NvmeCreateIoSubmissionQueue (Private);
 694   if (EFI_ERROR (Status)) {
 695     DEBUG ((DEBUG_ERROR, "%a: Create IO submission queue fail, Status - %r\n", __FUNCTION__, Status));
 696   }
 697
 698   return Status;
 699 }
 700
 701 /**
 702   Free the DMA resources allocated by an NVME controller.
 703
 704   @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 705
 706 **/
 707 VOID
 708 NvmeFreeDmaResource (
 709   IN PEI_NVME_CONTROLLER_PRIVATE_DATA  *Private
 710   )
 711 {
 712   ASSERT (Private != NULL);
 713
 714   if (Private->BufferMapping != NULL) {
 715     IoMmuFreeBuffer (
 716       NVME_MEM_MAX_PAGES,
 717       Private->Buffer,
 718       Private->BufferMapping
 719       );
 720   }
 721
 722   return;
 723 }