MdeModulePkg/Bus/Pci/NvmExpressPei/NvmExpressPeiHci.c

   1 /** @file
   2   The NvmExpressPei driver is used to manage non-volatile memory subsystem
   3   which follows NVM Express specification at PEI phase.
   4
   5   Copyright (c) 2018 - 2019, Intel Corporation. All rights reserved.<BR>
   6
   7   SPDX-License-Identifier: BSD-2-Clause-Patent
   8
   9 **/
  10
  11 #include "NvmExpressPei.h"
  12
  13 /**
  14   Transfer MMIO Data to memory.
  15
  16   @param[in,out] MemBuffer    Destination: Memory address.
  17   @param[in] MmioAddr         Source: MMIO address.
  18   @param[in] Size             Size for read.
  19
  20   @retval EFI_SUCCESS         MMIO read sucessfully.
  21
  22 **/
  23 EFI_STATUS
  24 NvmeMmioRead (
  25   IN OUT VOID *MemBuffer,
  26   IN     UINTN MmioAddr,
  27   IN     UINTN Size
  28   )
  29 {
  30   UINTN    Offset;
  31   UINT8    Data;
  32   UINT8    *Ptr;
  33
  34   // priority has adjusted
  35   switch (Size) {
  36     case 4:
  37       *((UINT32 *)MemBuffer) = MmioRead32 (MmioAddr);
  38       break;
  39
  40     case 8:
  41       *((UINT64 *)MemBuffer) = MmioRead64 (MmioAddr);
  42       break;
  43
  44     case 2:
  45       *((UINT16 *)MemBuffer) = MmioRead16 (MmioAddr);
  46       break;
  47
  48     case 1:
  49       *((UINT8 *)MemBuffer) = MmioRead8 (MmioAddr);
  50       break;
  51
  52     default:
  53       Ptr = (UINT8 *)MemBuffer;
  54       for (Offset = 0; Offset < Size; Offset += 1) {
  55         Data = MmioRead8 (MmioAddr + Offset);
  56         Ptr[Offset] = Data;
  57       }
  58       break;
  59   }
  60
  61   return EFI_SUCCESS;
  62 }
  63
  64 /**
  65   Transfer memory data to MMIO.
  66
  67   @param[in,out] MmioAddr    Destination: MMIO address.
  68   @param[in] MemBuffer       Source: Memory address.
  69   @param[in] Size            Size for write.
  70
  71   @retval EFI_SUCCESS        MMIO write sucessfully.
  72
  73 **/
  74 EFI_STATUS
  75 NvmeMmioWrite (
  76   IN OUT UINTN MmioAddr,
  77   IN     VOID *MemBuffer,
  78   IN     UINTN Size
  79   )
  80 {
  81   UINTN    Offset;
  82   UINT8    Data;
  83   UINT8    *Ptr;
  84
  85   // priority has adjusted
  86   switch (Size) {
  87     case 4:
  88       MmioWrite32 (MmioAddr, *((UINT32 *)MemBuffer));
  89       break;
  90
  91     case 8:
  92       MmioWrite64 (MmioAddr, *((UINT64 *)MemBuffer));
  93       break;
  94
  95     case 2:
  96       MmioWrite16 (MmioAddr, *((UINT16 *)MemBuffer));
  97       break;
  98
  99     case 1:
 100       MmioWrite8 (MmioAddr, *((UINT8 *)MemBuffer));
 101       break;
 102
 103     default:
 104       Ptr = (UINT8 *)MemBuffer;
 105       for (Offset = 0; Offset < Size; Offset += 1) {
 106         Data = Ptr[Offset];
 107         MmioWrite8 (MmioAddr + Offset, Data);
 108       }
 109       break;
 110   }
 111
 112   return EFI_SUCCESS;
 113 }
 114
 115 /**
 116   Get the page offset for specific NVME based memory.
 117
 118   @param[in] BaseMemIndex    The Index of BaseMem (0-based).
 119
 120   @retval - The page count for specific BaseMem Index
 121
 122 **/
 123 UINT32
 124 NvmeBaseMemPageOffset (
 125   IN UINTN              BaseMemIndex
 126   )
 127 {
 128   UINT32                Pages;
 129   UINTN                 Index;
 130   UINT32                PageSizeList[5];
 131
 132   PageSizeList[0] = 1;  /* ASQ */
 133   PageSizeList[1] = 1;  /* ACQ */
 134   PageSizeList[2] = 1;  /* SQs */
 135   PageSizeList[3] = 1;  /* CQs */
 136   PageSizeList[4] = NVME_PRP_SIZE;  /* PRPs */
 137
 138   if (BaseMemIndex > MAX_BASEMEM_COUNT) {
 139     DEBUG ((DEBUG_ERROR, "%a: The input BaseMem index is invalid.\n", __FUNCTION__));
 140     ASSERT (FALSE);
 141     return 0;
 142   }
 143
 144   Pages = 0;
 145   for (Index = 0; Index < BaseMemIndex; Index++) {
 146     Pages += PageSizeList[Index];
 147   }
 148
 149   return Pages;
 150 }
 151
 152 /**
 153   Wait for NVME controller status to be ready or not.
 154
 155   @param[in] Private      The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 156   @param[in] WaitReady    Flag for waitting status ready or not.
 157
 158   @return EFI_SUCCESS     Successfully to wait specific status.
 159   @return others          Fail to wait for specific controller status.
 160
 161 **/
 162 EFI_STATUS
 163 NvmeWaitController (
 164   IN PEI_NVME_CONTROLLER_PRIVATE_DATA    *Private,
 165   IN BOOLEAN                             WaitReady
 166   )
 167 {
 168   NVME_CSTS              Csts;
 169   EFI_STATUS             Status;
 170   UINT32                 Index;
 171   UINT8                  Timeout;
 172
 173   //
 174   // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to set after
 175   // Cc.Enable. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To.
 176   //
 177   if (Private->Cap.To == 0) {
 178     Timeout = 1;
 179   } else {
 180     Timeout = Private->Cap.To;
 181   }
 182
 183   Status = EFI_SUCCESS;
 184   for(Index = (Timeout * 500); Index != 0; --Index) {
 185     MicroSecondDelay (1000);
 186
 187     //
 188     // Check if the controller is initialized
 189     //
 190     Status = NVME_GET_CSTS (Private, &Csts);
 191     if (EFI_ERROR(Status)) {
 192       DEBUG ((DEBUG_ERROR, "%a: NVME_GET_CSTS fail, Status - %r\n", __FUNCTION__, Status));
 193       return Status;
 194     }
 195
 196     if ((BOOLEAN) Csts.Rdy == WaitReady) {
 197       break;
 198     }
 199   }
 200
 201   if (Index == 0) {
 202     Status = EFI_TIMEOUT;
 203   }
 204
 205   return Status;
 206 }
 207
 208 /**
 209   Disable the Nvm Express controller.
 210
 211   @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 212
 213   @return EFI_SUCCESS    Successfully disable the controller.
 214   @return others         Fail to disable the controller.
 215
 216 **/
 217 EFI_STATUS
 218 NvmeDisableController (
 219   IN PEI_NVME_CONTROLLER_PRIVATE_DATA    *Private
 220   )
 221 {
 222   NVME_CC       Cc;
 223   NVME_CSTS     Csts;
 224   EFI_STATUS    Status;
 225
 226   Status = NVME_GET_CSTS (Private, &Csts);
 227
 228   //
 229   // Read Controller Configuration Register.
 230   //
 231   Status = NVME_GET_CC (Private, &Cc);
 232   if (EFI_ERROR (Status)) {
 233     DEBUG ((DEBUG_ERROR, "%a: NVME_GET_CC fail, Status - %r\n", __FUNCTION__, Status));
 234     goto ErrorExit;
 235   }
 236
 237   if (Cc.En == 1) {
 238     Cc.En = 0;
 239     //
 240     // Disable the controller.
 241     //
 242     Status = NVME_SET_CC (Private, &Cc);
 243     if (EFI_ERROR (Status)) {
 244       DEBUG ((DEBUG_ERROR, "%a: NVME_SET_CC fail, Status - %r\n", __FUNCTION__, Status));
 245       goto ErrorExit;
 246     }
 247   }
 248
 249   Status = NvmeWaitController (Private, FALSE);
 250   if (EFI_ERROR (Status)) {
 251     DEBUG ((DEBUG_ERROR, "%a: NvmeWaitController fail, Status - %r\n", __FUNCTION__, Status));
 252     goto ErrorExit;
 253   }
 254
 255   return EFI_SUCCESS;
 256
 257 ErrorExit:
 258   DEBUG ((DEBUG_ERROR, "%a fail, Status - %r\n", __FUNCTION__, Status));
 259   return Status;
 260 }
 261
 262 /**
 263   Enable the Nvm Express controller.
 264
 265   @param[in] Private    The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 266
 267   @return EFI_SUCCESS         Successfully enable the controller.
 268   @return EFI_DEVICE_ERROR    Fail to enable the controller.
 269   @return EFI_TIMEOUT         Fail to enable the controller in given time slot.
 270
 271 **/
 272 EFI_STATUS
 273 NvmeEnableController (
 274   IN PEI_NVME_CONTROLLER_PRIVATE_DATA    *Private
 275   )
 276 {
 277   NVME_CC       Cc;
 278   EFI_STATUS    Status;
 279
 280   //
 281   // Enable the controller
 282   // CC.AMS, CC.MPS and CC.CSS are all set to 0
 283   //
 284   ZeroMem (&Cc, sizeof (NVME_CC));
 285   Cc.En     = 1;
 286   Cc.Iosqes = 6;
 287   Cc.Iocqes = 4;
 288   Status    = NVME_SET_CC (Private, &Cc);
 289   if (EFI_ERROR (Status)) {
 290     DEBUG ((DEBUG_ERROR, "%a: NVME_SET_CC fail, Status - %r\n", __FUNCTION__, Status));
 291     goto ErrorExit;
 292   }
 293
 294   Status = NvmeWaitController (Private, TRUE);
 295   if (EFI_ERROR (Status)) {
 296     DEBUG ((DEBUG_ERROR, "%a: NvmeWaitController fail, Status - %r\n", __FUNCTION__, Status));
 297     goto ErrorExit;
 298   }
 299
 300   return EFI_SUCCESS;
 301
 302 ErrorExit:
 303   DEBUG ((DEBUG_ERROR, "%a fail, Status: %r\n", __FUNCTION__, Status));
 304   return Status;
 305 }
 306
 307 /**
 308   Get the Identify Controller data.
 309
 310   @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 311   @param[in] Buffer      The Buffer used to store the Identify Controller data.
 312
 313   @return EFI_SUCCESS    Successfully get the Identify Controller data.
 314   @return others         Fail to get the Identify Controller data.
 315
 316 **/
 317 EFI_STATUS
 318 NvmeIdentifyController (
 319   IN PEI_NVME_CONTROLLER_PRIVATE_DATA    *Private,
 320   IN VOID                                *Buffer
 321   )
 322 {
 323   EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET    CommandPacket;
 324   EFI_NVM_EXPRESS_COMMAND                     Command;
 325   EFI_NVM_EXPRESS_COMPLETION                  Completion;
 326   EFI_STATUS                                  Status;
 327
 328   ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
 329   ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND));
 330   ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION));
 331
 332   Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD;
 333   //
 334   // According to Nvm Express 1.1 spec Figure 38, When not used, the field shall be cleared to 0h.
 335   // For the Identify command, the Namespace Identifier is only used for the Namespace Data structure.
 336   //
 337   Command.Nsid        = 0;
 338
 339   CommandPacket.NvmeCmd        = &Command;
 340   CommandPacket.NvmeCompletion = &Completion;
 341   CommandPacket.TransferBuffer = Buffer;
 342   CommandPacket.TransferLength = sizeof (NVME_ADMIN_CONTROLLER_DATA);
 343   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
 344   CommandPacket.QueueType      = NVME_ADMIN_QUEUE;
 345   //
 346   // Set bit 0 (Cns bit) to 1 to identify the controller
 347   //
 348   CommandPacket.NvmeCmd->Cdw10 = 1;
 349   CommandPacket.NvmeCmd->Flags = CDW10_VALID;
 350
 351   Status = NvmePassThruExecute (
 352              Private,
 353              NVME_CONTROLLER_NSID,
 354              &CommandPacket
 355              );
 356   return Status;
 357 }
 358
 359 /**
 360   Get specified identify namespace data.
 361
 362   @param[in] Private        The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 363   @param[in] NamespaceId    The specified namespace identifier.
 364   @param[in] Buffer         The buffer used to store the identify namespace data.
 365
 366   @return EFI_SUCCESS         Successfully get the identify namespace data.
 367   @return EFI_DEVICE_ERROR    Fail to get the identify namespace data.
 368
 369 **/
 370 EFI_STATUS
 371 NvmeIdentifyNamespace (
 372   IN PEI_NVME_CONTROLLER_PRIVATE_DATA    *Private,
 373   IN UINT32                              NamespaceId,
 374   IN VOID                                *Buffer
 375   )
 376 {
 377   EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET    CommandPacket;
 378   EFI_NVM_EXPRESS_COMMAND                     Command;
 379   EFI_NVM_EXPRESS_COMPLETION                  Completion;
 380   EFI_STATUS                                  Status;
 381
 382   ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
 383   ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND));
 384   ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION));
 385
 386   Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD;
 387   Command.Nsid        = NamespaceId;
 388
 389   CommandPacket.NvmeCmd        = &Command;
 390   CommandPacket.NvmeCompletion = &Completion;
 391   CommandPacket.TransferBuffer = Buffer;
 392   CommandPacket.TransferLength = sizeof (NVME_ADMIN_NAMESPACE_DATA);
 393   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
 394   CommandPacket.QueueType      = NVME_ADMIN_QUEUE;
 395   //
 396   // Set bit 0 (Cns bit) to 1 to identify a namespace
 397   //
 398   CommandPacket.NvmeCmd->Cdw10 = 0;
 399   CommandPacket.NvmeCmd->Flags = CDW10_VALID;
 400
 401   Status = NvmePassThruExecute (
 402              Private,
 403              NamespaceId,
 404              &CommandPacket
 405              );
 406   return Status;
 407 }
 408
 409 /**
 410   Dump the Identify Controller data.
 411
 412   @param[in] ControllerData    The pointer to the NVME_ADMIN_CONTROLLER_DATA data structure.
 413
 414 **/
 415 VOID
 416 NvmeDumpControllerData (
 417   IN NVME_ADMIN_CONTROLLER_DATA    *ControllerData
 418   )
 419 {
 420   UINT8    Sn[21];
 421   UINT8    Mn[41];
 422
 423   CopyMem (Sn, ControllerData->Sn, sizeof (ControllerData->Sn));
 424   Sn[20] = 0;
 425   CopyMem (Mn, ControllerData->Mn, sizeof (ControllerData->Mn));
 426   Mn[40] = 0;
 427
 428   DEBUG ((DEBUG_INFO, " == NVME IDENTIFY CONTROLLER DATA ==\n"));
 429   DEBUG ((DEBUG_INFO, "    PCI VID   : 0x%x\n", ControllerData->Vid));
 430   DEBUG ((DEBUG_INFO, "    PCI SSVID : 0x%x\n", ControllerData->Ssvid));
 431   DEBUG ((DEBUG_INFO, "    SN        : %a\n",   Sn));
 432   DEBUG ((DEBUG_INFO, "    MN        : %a\n",   Mn));
 433   DEBUG ((DEBUG_INFO, "    FR        : 0x%lx\n", *((UINT64*)ControllerData->Fr)));
 434   DEBUG ((DEBUG_INFO, "    RAB       : 0x%x\n", ControllerData->Rab));
 435   DEBUG ((DEBUG_INFO, "    IEEE      : 0x%x\n", *(UINT32*)ControllerData->Ieee_oui));
 436   DEBUG ((DEBUG_INFO, "    AERL      : 0x%x\n", ControllerData->Aerl));
 437   DEBUG ((DEBUG_INFO, "    SQES      : 0x%x\n", ControllerData->Sqes));
 438   DEBUG ((DEBUG_INFO, "    CQES      : 0x%x\n", ControllerData->Cqes));
 439   DEBUG ((DEBUG_INFO, "    NN        : 0x%x\n", ControllerData->Nn));
 440   return;
 441 }
 442
 443 /**
 444   Create IO completion queue.
 445
 446   @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 447
 448   @return EFI_SUCCESS    Successfully create io completion queue.
 449   @return others         Fail to create io completion queue.
 450
 451 **/
 452 EFI_STATUS
 453 NvmeCreateIoCompletionQueue (
 454   IN PEI_NVME_CONTROLLER_PRIVATE_DATA    *Private
 455   )
 456 {
 457   EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET    CommandPacket;
 458   EFI_NVM_EXPRESS_COMMAND                     Command;
 459   EFI_NVM_EXPRESS_COMPLETION                  Completion;
 460   EFI_STATUS                                  Status;
 461   NVME_ADMIN_CRIOCQ                           CrIoCq;
 462
 463   ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
 464   ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND));
 465   ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION));
 466   ZeroMem (&CrIoCq, sizeof(NVME_ADMIN_CRIOCQ));
 467
 468   CommandPacket.NvmeCmd        = &Command;
 469   CommandPacket.NvmeCompletion = &Completion;
 470
 471   Command.Cdw0.Opcode = NVME_ADMIN_CRIOCQ_CMD;
 472   CommandPacket.TransferBuffer = Private->CqBuffer[NVME_IO_QUEUE];
 473   CommandPacket.TransferLength = EFI_PAGE_SIZE;
 474   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
 475   CommandPacket.QueueType      = NVME_ADMIN_QUEUE;
 476
 477   CrIoCq.Qid   = NVME_IO_QUEUE;
 478   CrIoCq.Qsize = NVME_CCQ_SIZE;
 479   CrIoCq.Pc    = 1;
 480   CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoCq, sizeof (NVME_ADMIN_CRIOCQ));
 481   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
 482
 483   Status = NvmePassThruExecute (
 484              Private,
 485              NVME_CONTROLLER_NSID,
 486              &CommandPacket
 487              );
 488   return Status;
 489 }
 490
 491 /**
 492   Create IO submission queue.
 493
 494   @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 495
 496   @return EFI_SUCCESS    Successfully create io submission queue.
 497   @return others         Fail to create io submission queue.
 498
 499 **/
 500 EFI_STATUS
 501 NvmeCreateIoSubmissionQueue (
 502   IN PEI_NVME_CONTROLLER_PRIVATE_DATA    *Private
 503   )
 504 {
 505   EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET    CommandPacket;
 506   EFI_NVM_EXPRESS_COMMAND                     Command;
 507   EFI_NVM_EXPRESS_COMPLETION                  Completion;
 508   EFI_STATUS                                  Status;
 509   NVME_ADMIN_CRIOSQ                           CrIoSq;
 510
 511   ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
 512   ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND));
 513   ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION));
 514   ZeroMem (&CrIoSq, sizeof(NVME_ADMIN_CRIOSQ));
 515
 516   CommandPacket.NvmeCmd        = &Command;
 517   CommandPacket.NvmeCompletion = &Completion;
 518
 519   Command.Cdw0.Opcode = NVME_ADMIN_CRIOSQ_CMD;
 520   CommandPacket.TransferBuffer = Private->SqBuffer[NVME_IO_QUEUE];
 521   CommandPacket.TransferLength = EFI_PAGE_SIZE;
 522   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
 523   CommandPacket.QueueType      = NVME_ADMIN_QUEUE;
 524
 525   CrIoSq.Qid   = NVME_IO_QUEUE;
 526   CrIoSq.Qsize = NVME_CSQ_SIZE;
 527   CrIoSq.Pc    = 1;
 528   CrIoSq.Cqid  = NVME_IO_QUEUE;
 529   CrIoSq.Qprio = 0;
 530   CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoSq, sizeof (NVME_ADMIN_CRIOSQ));
 531   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
 532
 533   Status = NvmePassThruExecute (
 534              Private,
 535              NVME_CONTROLLER_NSID,
 536              &CommandPacket
 537              );
 538   return Status;
 539 }
 540
 541 /**
 542   Initialize the Nvm Express controller.
 543
 544   @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 545
 546   @retval EFI_SUCCESS    The NVM Express Controller is initialized successfully.
 547   @retval Others         A device error occurred while initializing the controller.
 548
 549 **/
 550 EFI_STATUS
 551 NvmeControllerInit (
 552   IN PEI_NVME_CONTROLLER_PRIVATE_DATA    *Private
 553   )
 554 {
 555   EFI_STATUS    Status;
 556   UINTN         Index;
 557   NVME_AQA      Aqa;
 558   NVME_ASQ      Asq;
 559   NVME_ACQ      Acq;
 560   NVME_VER      Ver;
 561
 562   //
 563   // Dump the NVME controller implementation version
 564   //
 565   NVME_GET_VER (Private, &Ver);
 566   DEBUG ((DEBUG_INFO, "NVME controller implementation version: %d.%d\n", Ver.Mjr, Ver.Mnr));
 567
 568   //
 569   // Read the controller Capabilities register and verify that the NVM command set is supported
 570   //
 571   NVME_GET_CAP (Private, &Private->Cap);
 572   if (Private->Cap.Css != 0x01) {
 573     DEBUG ((DEBUG_ERROR, "%a: The NVME controller doesn't support NVMe command set.\n", __FUNCTION__));
 574     return EFI_UNSUPPORTED;
 575   }
 576
 577   //
 578   // Currently, the driver only supports 4k page size
 579   //
 580   if ((Private->Cap.Mpsmin + 12) > EFI_PAGE_SHIFT) {
 581     DEBUG ((DEBUG_ERROR, "%a: The driver doesn't support page size other than 4K.\n", __FUNCTION__));
 582     ASSERT (FALSE);
 583     return EFI_UNSUPPORTED;
 584   }
 585
 586   for (Index = 0; Index < NVME_MAX_QUEUES; Index++) {
 587     Private->Pt[Index]  = 0;
 588     Private->Cid[Index] = 0;
 589     ZeroMem ((VOID *)(UINTN)(&Private->SqTdbl[Index]), sizeof (NVME_SQTDBL));
 590     ZeroMem ((VOID *)(UINTN)(&Private->CqHdbl[Index]), sizeof (NVME_CQHDBL));
 591   }
 592   ZeroMem (Private->Buffer, EFI_PAGE_SIZE * NVME_MEM_MAX_PAGES);
 593
 594   //
 595   // Disable the NVME controller first
 596   //
 597   Status = NvmeDisableController (Private);
 598   if (EFI_ERROR (Status)) {
 599     DEBUG ((DEBUG_ERROR, "%a: NvmeDisableController fail, Status - %r\n", __FUNCTION__, Status));
 600     return Status;
 601   }
 602
 603   //
 604   // Set the number of entries in admin submission & completion queues
 605   //
 606   Aqa.Asqs  = NVME_ASQ_SIZE;
 607   Aqa.Rsvd1 = 0;
 608   Aqa.Acqs  = NVME_ACQ_SIZE;
 609   Aqa.Rsvd2 = 0;
 610
 611   //
 612   // Address of admin submission & completion queues
 613   //
 614   Asq = (UINT64)(UINTN)(NVME_ASQ_BASE (Private) & ~0xFFF);
 615   Acq = (UINT64)(UINTN)(NVME_ACQ_BASE (Private) & ~0xFFF);
 616
 617   //
 618   // Address of I/O submission & completion queues
 619   //
 620   Private->SqBuffer[0] = (NVME_SQ *)(UINTN)NVME_ASQ_BASE (Private);    // NVME_ADMIN_QUEUE
 621   Private->CqBuffer[0] = (NVME_CQ *)(UINTN)NVME_ACQ_BASE (Private);    // NVME_ADMIN_QUEUE
 622   Private->SqBuffer[1] = (NVME_SQ *)(UINTN)NVME_SQ_BASE (Private, 0);  // NVME_IO_QUEUE
 623   Private->CqBuffer[1] = (NVME_CQ *)(UINTN)NVME_CQ_BASE (Private, 0);  // NVME_IO_QUEUE
 624   DEBUG ((DEBUG_INFO, "Admin Submission Queue Size (Aqa.Asqs) = [%08X]\n", Aqa.Asqs));
 625   DEBUG ((DEBUG_INFO, "Admin Completion Queue Size (Aqa.Acqs) = [%08X]\n", Aqa.Acqs));
 626   DEBUG ((DEBUG_INFO, "Admin Submission Queue (SqBuffer[0]) =   [%08X]\n", Private->SqBuffer[0]));
 627   DEBUG ((DEBUG_INFO, "Admin Completion Queue (CqBuffer[0]) =   [%08X]\n", Private->CqBuffer[0]));
 628   DEBUG ((DEBUG_INFO, "I/O   Submission Queue (SqBuffer[1]) =   [%08X]\n", Private->SqBuffer[1]));
 629   DEBUG ((DEBUG_INFO, "I/O   Completion Queue (CqBuffer[1]) =   [%08X]\n", Private->CqBuffer[1]));
 630
 631   //
 632   // Program admin queue attributes
 633   //
 634   NVME_SET_AQA (Private, &Aqa);
 635
 636   //
 637   // Program admin submission & completion queues address
 638   //
 639   NVME_SET_ASQ (Private, &Asq);
 640   NVME_SET_ACQ (Private, &Acq);
 641
 642   //
 643   // Enable the NVME controller
 644   //
 645   Status = NvmeEnableController (Private);
 646   if (EFI_ERROR (Status)) {
 647     DEBUG ((DEBUG_ERROR, "%a: NvmeEnableController fail, Status - %r\n", __FUNCTION__, Status));
 648     return Status;
 649   }
 650
 651   //
 652   // Get the Identify Controller data
 653   //
 654   if (Private->ControllerData == NULL) {
 655     Private->ControllerData = (NVME_ADMIN_CONTROLLER_DATA *)AllocateZeroPool (sizeof (NVME_ADMIN_CONTROLLER_DATA));
 656     if (Private->ControllerData == NULL) {
 657       return EFI_OUT_OF_RESOURCES;
 658     }
 659   }
 660   Status = NvmeIdentifyController (Private, Private->ControllerData);
 661   if (EFI_ERROR (Status)) {
 662     DEBUG ((DEBUG_ERROR, "%a: NvmeIdentifyController fail, Status - %r\n", __FUNCTION__, Status));
 663     return Status;
 664   }
 665   NvmeDumpControllerData (Private->ControllerData);
 666
 667   //
 668   // Check the namespace number for storing the namespaces information
 669   //
 670   if (Private->ControllerData->Nn > MAX_UINT32 / sizeof (PEI_NVME_NAMESPACE_INFO)) {
 671     DEBUG ((
 672       DEBUG_ERROR,
 673       "%a: Number of Namespaces field in Identify Controller data not supported by the driver.\n",
 674       __FUNCTION__
 675       ));
 676     return EFI_UNSUPPORTED;
 677   }
 678
 679   //
 680   // Create one I/O completion queue and one I/O submission queue
 681   //
 682   Status = NvmeCreateIoCompletionQueue (Private);
 683   if (EFI_ERROR (Status)) {
 684     DEBUG ((DEBUG_ERROR, "%a: Create IO completion queue fail, Status - %r\n", __FUNCTION__, Status));
 685     return Status;
 686   }
 687   Status = NvmeCreateIoSubmissionQueue (Private);
 688   if (EFI_ERROR (Status)) {
 689     DEBUG ((DEBUG_ERROR, "%a: Create IO submission queue fail, Status - %r\n", __FUNCTION__, Status));
 690   }
 691
 692   return Status;
 693 }
 694
 695 /**
 696   Free the DMA resources allocated by an NVME controller.
 697
 698   @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 699
 700 **/
 701 VOID
 702 NvmeFreeDmaResource (
 703   IN PEI_NVME_CONTROLLER_PRIVATE_DATA    *Private
 704   )
 705 {
 706   ASSERT (Private != NULL);
 707
 708   if (Private->BufferMapping != NULL) {
 709     IoMmuFreeBuffer (
 710        NVME_MEM_MAX_PAGES,
 711        Private->Buffer,
 712        Private->BufferMapping
 713        );
 714   }
 715
 716   return;
 717 }