MdeModulePkg/Bus/Pci/NvmExpressPei/NvmExpressPeiHci.c

   1 /** @file
   2   The NvmExpressPei driver is used to manage non-volatile memory subsystem
   3   which follows NVM Express specification at PEI phase.
   4
   5   Copyright (c) 2018 - 2019, Intel Corporation. All rights reserved.<BR>
   6
   7   SPDX-License-Identifier: BSD-2-Clause-Patent
   8
   9 **/
  10
  11 #include "NvmExpressPei.h"
  12
  13 /**
  14   Transfer MMIO Data to memory.
  15
  16   @param[in,out] MemBuffer    Destination: Memory address.
  17   @param[in] MmioAddr         Source: MMIO address.
  18   @param[in] Size             Size for read.
  19
  20   @retval EFI_SUCCESS         MMIO read sucessfully.
  21
  22 **/
  23 EFI_STATUS
  24 NvmeMmioRead (
  25   IN OUT VOID *MemBuffer,
  26   IN     UINTN MmioAddr,
  27   IN     UINTN Size
  28   )
  29 {
  30   UINTN    Offset;
  31   UINT8    Data;
  32   UINT8    *Ptr;
  33
  34   // priority has adjusted
  35   switch (Size) {
  36     case 4:
  37       *((UINT32 *)MemBuffer) = MmioRead32 (MmioAddr);
  38       break;
  39
  40     case 8:
  41       *((UINT64 *)MemBuffer) = MmioRead64 (MmioAddr);
  42       break;
  43
  44     case 2:
  45       *((UINT16 *)MemBuffer) = MmioRead16 (MmioAddr);
  46       break;
  47
  48     case 1:
  49       *((UINT8 *)MemBuffer) = MmioRead8 (MmioAddr);
  50       break;
  51
  52     default:
  53       Ptr = (UINT8 *)MemBuffer;
  54       for (Offset = 0; Offset < Size; Offset += 1) {
  55         Data = MmioRead8 (MmioAddr + Offset);
  56         Ptr[Offset] = Data;
  57       }
  58       break;
  59   }
  60
  61   return EFI_SUCCESS;
  62 }
  63
  64 /**
  65   Transfer memory data to MMIO.
  66
  67   @param[in,out] MmioAddr    Destination: MMIO address.
  68   @param[in] MemBuffer       Source: Memory address.
  69   @param[in] Size            Size for write.
  70
  71   @retval EFI_SUCCESS        MMIO write sucessfully.
  72
  73 **/
  74 EFI_STATUS
  75 NvmeMmioWrite (
  76   IN OUT UINTN MmioAddr,
  77   IN     VOID *MemBuffer,
  78   IN     UINTN Size
  79   )
  80 {
  81   UINTN    Offset;
  82   UINT8    Data;
  83   UINT8    *Ptr;
  84
  85   // priority has adjusted
  86   switch (Size) {
  87     case 4:
  88       MmioWrite32 (MmioAddr, *((UINT32 *)MemBuffer));
  89       break;
  90
  91     case 8:
  92       MmioWrite64 (MmioAddr, *((UINT64 *)MemBuffer));
  93       break;
  94
  95     case 2:
  96       MmioWrite16 (MmioAddr, *((UINT16 *)MemBuffer));
  97       break;
  98
  99     case 1:
 100       MmioWrite8 (MmioAddr, *((UINT8 *)MemBuffer));
 101       break;
 102
 103     default:
 104       Ptr = (UINT8 *)MemBuffer;
 105       for (Offset = 0; Offset < Size; Offset += 1) {
 106         Data = Ptr[Offset];
 107         MmioWrite8 (MmioAddr + Offset, Data);
 108       }
 109       break;
 110   }
 111
 112   return EFI_SUCCESS;
 113 }
 114
 115 /**
 116   Get the page offset for specific NVME based memory.
 117
 118   @param[in] BaseMemIndex    The Index of BaseMem (0-based).
 119
 120   @retval - The page count for specific BaseMem Index
 121
 122 **/
 123 UINT32
 124 NvmeBaseMemPageOffset (
 125   IN UINTN              BaseMemIndex
 126   )
 127 {
 128   UINT32                Pages;
 129   UINTN                 Index;
 130   UINT32                PageSizeList[5];
 131
 132   PageSizeList[0] = 1;  /* ASQ */
 133   PageSizeList[1] = 1;  /* ACQ */
 134   PageSizeList[2] = 1;  /* SQs */
 135   PageSizeList[3] = 1;  /* CQs */
 136   PageSizeList[4] = NVME_PRP_SIZE;  /* PRPs */
 137
 138   if (BaseMemIndex > MAX_BASEMEM_COUNT) {
 139     DEBUG ((DEBUG_ERROR, "%a: The input BaseMem index is invalid.\n", __FUNCTION__));
 140     ASSERT (FALSE);
 141     return 0;
 142   }
 143
 144   Pages = 0;
 145   for (Index = 0; Index < BaseMemIndex; Index++) {
 146     Pages += PageSizeList[Index];
 147   }
 148
 149   return Pages;
 150 }
 151
 152 /**
 153   Wait for NVME controller status to be ready or not.
 154
 155   @param[in] Private      The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 156   @param[in] WaitReady    Flag for waitting status ready or not.
 157
 158   @return EFI_SUCCESS     Successfully to wait specific status.
 159   @return others          Fail to wait for specific controller status.
 160
 161 **/
 162 EFI_STATUS
 163 NvmeWaitController (
 164   IN PEI_NVME_CONTROLLER_PRIVATE_DATA    *Private,
 165   IN BOOLEAN                             WaitReady
 166   )
 167 {
 168   NVME_CSTS              Csts;
 169   EFI_STATUS             Status;
 170   UINT32                 Index;
 171   UINT8                  Timeout;
 172
 173   //
 174   // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to set after
 175   // Cc.Enable. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To.
 176   //
 177   if (Private->Cap.To == 0) {
 178     Timeout = 1;
 179   } else {
 180     Timeout = Private->Cap.To;
 181   }
 182
 183   Status = EFI_SUCCESS;
 184   for(Index = (Timeout * 500); Index != 0; --Index) {
 185     MicroSecondDelay (1000);
 186
 187     //
 188     // Check if the controller is initialized
 189     //
 190     Status = NVME_GET_CSTS (Private, &Csts);
 191     if (EFI_ERROR(Status)) {
 192       DEBUG ((DEBUG_ERROR, "%a: NVME_GET_CSTS fail, Status - %r\n", __FUNCTION__, Status));
 193       return Status;
 194     }
 195
 196     if ((BOOLEAN) Csts.Rdy == WaitReady) {
 197       break;
 198     }
 199   }
 200
 201   if (Index == 0) {
 202     Status = EFI_TIMEOUT;
 203   }
 204
 205   return Status;
 206 }
 207
 208 /**
 209   Disable the Nvm Express controller.
 210
 211   @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 212
 213   @return EFI_SUCCESS    Successfully disable the controller.
 214   @return others         Fail to disable the controller.
 215
 216 **/
 217 EFI_STATUS
 218 NvmeDisableController (
 219   IN PEI_NVME_CONTROLLER_PRIVATE_DATA    *Private
 220   )
 221 {
 222   NVME_CC       Cc;
 223   NVME_CSTS     Csts;
 224   EFI_STATUS    Status;
 225
 226   Status = NVME_GET_CSTS (Private, &Csts);
 227
 228   //
 229   // Read Controller Configuration Register.
 230   //
 231   Status = NVME_GET_CC (Private, &Cc);
 232   if (EFI_ERROR (Status)) {
 233     DEBUG ((DEBUG_ERROR, "%a: NVME_GET_CC fail, Status - %r\n", __FUNCTION__, Status));
 234     goto ErrorExit;
 235   }
 236
 237   if (Cc.En == 1) {
 238     Cc.En = 0;
 239     //
 240     // Disable the controller.
 241     //
 242     Status = NVME_SET_CC (Private, &Cc);
 243     if (EFI_ERROR (Status)) {
 244       DEBUG ((DEBUG_ERROR, "%a: NVME_SET_CC fail, Status - %r\n", __FUNCTION__, Status));
 245       goto ErrorExit;
 246     }
 247   }
 248
 249   Status = NvmeWaitController (Private, FALSE);
 250   if (EFI_ERROR (Status)) {
 251     DEBUG ((DEBUG_ERROR, "%a: NvmeWaitController fail, Status - %r\n", __FUNCTION__, Status));
 252     goto ErrorExit;
 253   }
 254
 255   return EFI_SUCCESS;
 256
 257 ErrorExit:
 258   DEBUG ((DEBUG_ERROR, "%a fail, Status - %r\n", __FUNCTION__, Status));
 259   return Status;
 260 }
 261
 262 /**
 263   Enable the Nvm Express controller.
 264
 265   @param[in] Private    The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 266
 267   @return EFI_SUCCESS         Successfully enable the controller.
 268   @return EFI_DEVICE_ERROR    Fail to enable the controller.
 269   @return EFI_TIMEOUT         Fail to enable the controller in given time slot.
 270
 271 **/
 272 EFI_STATUS
 273 NvmeEnableController (
 274   IN PEI_NVME_CONTROLLER_PRIVATE_DATA    *Private
 275   )
 276 {
 277   NVME_CC       Cc;
 278   EFI_STATUS    Status;
 279
 280   //
 281   // Enable the controller
 282   // CC.AMS, CC.MPS and CC.CSS are all set to 0
 283   //
 284   ZeroMem (&Cc, sizeof (NVME_CC));
 285   Cc.En     = 1;
 286   Cc.Iosqes = 6;
 287   Cc.Iocqes = 4;
 288   Status    = NVME_SET_CC (Private, &Cc);
 289   if (EFI_ERROR (Status)) {
 290     DEBUG ((DEBUG_ERROR, "%a: NVME_SET_CC fail, Status - %r\n", __FUNCTION__, Status));
 291     goto ErrorExit;
 292   }
 293
 294   Status = NvmeWaitController (Private, TRUE);
 295   if (EFI_ERROR (Status)) {
 296     DEBUG ((DEBUG_ERROR, "%a: NvmeWaitController fail, Status - %r\n", __FUNCTION__, Status));
 297     goto ErrorExit;
 298   }
 299
 300   return EFI_SUCCESS;
 301
 302 ErrorExit:
 303   DEBUG ((DEBUG_ERROR, "%a fail, Status: %r\n", __FUNCTION__, Status));
 304   return Status;
 305 }
 306
 307 /**
 308   Get the Identify Controller data.
 309
 310   @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 311   @param[in] Buffer      The Buffer used to store the Identify Controller data.
 312
 313   @return EFI_SUCCESS    Successfully get the Identify Controller data.
 314   @return others         Fail to get the Identify Controller data.
 315
 316 **/
 317 EFI_STATUS
 318 NvmeIdentifyController (
 319   IN PEI_NVME_CONTROLLER_PRIVATE_DATA    *Private,
 320   IN VOID                                *Buffer
 321   )
 322 {
 323   EDKII_PEI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET    CommandPacket;
 324   EDKII_PEI_NVM_EXPRESS_COMMAND                     Command;
 325   EDKII_PEI_NVM_EXPRESS_COMPLETION                  Completion;
 326   EFI_STATUS                                        Status;
 327
 328   ZeroMem (&CommandPacket, sizeof(EDKII_PEI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
 329   ZeroMem (&Command, sizeof(EDKII_PEI_NVM_EXPRESS_COMMAND));
 330   ZeroMem (&Completion, sizeof(EDKII_PEI_NVM_EXPRESS_COMPLETION));
 331
 332   Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD;
 333   //
 334   // According to Nvm Express 1.1 spec Figure 38, When not used, the field shall be cleared to 0h.
 335   // For the Identify command, the Namespace Identifier is only used for the Namespace Data structure.
 336   //
 337   Command.Nsid        = 0;
 338
 339   CommandPacket.NvmeCmd        = &Command;
 340   CommandPacket.NvmeCompletion = &Completion;
 341   CommandPacket.TransferBuffer = Buffer;
 342   CommandPacket.TransferLength = sizeof (NVME_ADMIN_CONTROLLER_DATA);
 343   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
 344   CommandPacket.QueueType      = NVME_ADMIN_QUEUE;
 345   //
 346   // Set bit 0 (Cns bit) to 1 to identify the controller
 347   //
 348   CommandPacket.NvmeCmd->Cdw10 = 1;
 349   CommandPacket.NvmeCmd->Flags = CDW10_VALID;
 350
 351   Status = NvmePassThru (
 352              Private,
 353              NVME_CONTROLLER_NSID,
 354              &CommandPacket
 355              );
 356   return Status;
 357 }
 358
 359 /**
 360   Get specified identify namespace data.
 361
 362   @param[in] Private        The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 363   @param[in] NamespaceId    The specified namespace identifier.
 364   @param[in] Buffer         The buffer used to store the identify namespace data.
 365
 366   @return EFI_SUCCESS         Successfully get the identify namespace data.
 367   @return EFI_DEVICE_ERROR    Fail to get the identify namespace data.
 368
 369 **/
 370 EFI_STATUS
 371 NvmeIdentifyNamespace (
 372   IN PEI_NVME_CONTROLLER_PRIVATE_DATA    *Private,
 373   IN UINT32                              NamespaceId,
 374   IN VOID                                *Buffer
 375   )
 376 {
 377   EDKII_PEI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET    CommandPacket;
 378   EDKII_PEI_NVM_EXPRESS_COMMAND                     Command;
 379   EDKII_PEI_NVM_EXPRESS_COMPLETION                  Completion;
 380   EFI_STATUS                                        Status;
 381
 382   ZeroMem (&CommandPacket, sizeof(EDKII_PEI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
 383   ZeroMem (&Command, sizeof(EDKII_PEI_NVM_EXPRESS_COMMAND));
 384   ZeroMem (&Completion, sizeof(EDKII_PEI_NVM_EXPRESS_COMPLETION));
 385
 386   Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD;
 387   Command.Nsid        = NamespaceId;
 388
 389   CommandPacket.NvmeCmd        = &Command;
 390   CommandPacket.NvmeCompletion = &Completion;
 391   CommandPacket.TransferBuffer = Buffer;
 392   CommandPacket.TransferLength = sizeof (NVME_ADMIN_NAMESPACE_DATA);
 393   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
 394   CommandPacket.QueueType      = NVME_ADMIN_QUEUE;
 395   //
 396   // Set bit 0 (Cns bit) to 1 to identify a namespace
 397   //
 398   CommandPacket.NvmeCmd->Cdw10 = 0;
 399   CommandPacket.NvmeCmd->Flags = CDW10_VALID;
 400
 401   Status = NvmePassThru (
 402              Private,
 403              NamespaceId,
 404              &CommandPacket
 405              );
 406   return Status;
 407 }
 408
 409 /**
 410   Dump the Identify Controller data.
 411
 412   @param[in] ControllerData    The pointer to the NVME_ADMIN_CONTROLLER_DATA data structure.
 413
 414 **/
 415 VOID
 416 NvmeDumpControllerData (
 417   IN NVME_ADMIN_CONTROLLER_DATA    *ControllerData
 418   )
 419 {
 420   UINT8    Sn[21];
 421   UINT8    Mn[41];
 422
 423   CopyMem (Sn, ControllerData->Sn, sizeof (ControllerData->Sn));
 424   Sn[20] = 0;
 425   CopyMem (Mn, ControllerData->Mn, sizeof (ControllerData->Mn));
 426   Mn[40] = 0;
 427
 428   DEBUG ((DEBUG_INFO, " == NVME IDENTIFY CONTROLLER DATA ==\n"));
 429   DEBUG ((DEBUG_INFO, "    PCI VID   : 0x%x\n", ControllerData->Vid));
 430   DEBUG ((DEBUG_INFO, "    PCI SSVID : 0x%x\n", ControllerData->Ssvid));
 431   DEBUG ((DEBUG_INFO, "    SN        : %a\n",   Sn));
 432   DEBUG ((DEBUG_INFO, "    MN        : %a\n",   Mn));
 433   DEBUG ((DEBUG_INFO, "    FR        : 0x%lx\n", *((UINT64*)ControllerData->Fr)));
 434   DEBUG ((DEBUG_INFO, "    RAB       : 0x%x\n", ControllerData->Rab));
 435   DEBUG ((DEBUG_INFO, "    IEEE      : 0x%x\n", *(UINT32*)ControllerData->Ieee_oui));
 436   DEBUG ((DEBUG_INFO, "    AERL      : 0x%x\n", ControllerData->Aerl));
 437   DEBUG ((DEBUG_INFO, "    SQES      : 0x%x\n", ControllerData->Sqes));
 438   DEBUG ((DEBUG_INFO, "    CQES      : 0x%x\n", ControllerData->Cqes));
 439   DEBUG ((DEBUG_INFO, "    NN        : 0x%x\n", ControllerData->Nn));
 440   return;
 441 }
 442
 443 /**
 444   Create IO completion queue.
 445
 446   @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 447
 448   @return EFI_SUCCESS    Successfully create io completion queue.
 449   @return others         Fail to create io completion queue.
 450
 451 **/
 452 EFI_STATUS
 453 NvmeCreateIoCompletionQueue (
 454   IN PEI_NVME_CONTROLLER_PRIVATE_DATA    *Private
 455   )
 456 {
 457   EDKII_PEI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET    CommandPacket;
 458   EDKII_PEI_NVM_EXPRESS_COMMAND                     Command;
 459   EDKII_PEI_NVM_EXPRESS_COMPLETION                  Completion;
 460   EFI_STATUS                                        Status;
 461   NVME_ADMIN_CRIOCQ                                 CrIoCq;
 462
 463   ZeroMem (&CommandPacket, sizeof(EDKII_PEI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
 464   ZeroMem (&Command, sizeof(EDKII_PEI_NVM_EXPRESS_COMMAND));
 465   ZeroMem (&Completion, sizeof(EDKII_PEI_NVM_EXPRESS_COMPLETION));
 466   ZeroMem (&CrIoCq, sizeof(NVME_ADMIN_CRIOCQ));
 467
 468   CommandPacket.NvmeCmd        = &Command;
 469   CommandPacket.NvmeCompletion = &Completion;
 470
 471   Command.Cdw0.Opcode = NVME_ADMIN_CRIOCQ_CMD;
 472   Command.Cdw0.Cid    = Private->Cid[NVME_ADMIN_QUEUE]++;
 473   CommandPacket.TransferBuffer = Private->CqBuffer[NVME_IO_QUEUE];
 474   CommandPacket.TransferLength = EFI_PAGE_SIZE;
 475   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
 476   CommandPacket.QueueType      = NVME_ADMIN_QUEUE;
 477
 478   CrIoCq.Qid   = NVME_IO_QUEUE;
 479   CrIoCq.Qsize = NVME_CCQ_SIZE;
 480   CrIoCq.Pc    = 1;
 481   CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoCq, sizeof (NVME_ADMIN_CRIOCQ));
 482   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
 483
 484   Status = NvmePassThru (
 485              Private,
 486              NVME_CONTROLLER_NSID,
 487              &CommandPacket
 488              );
 489   return Status;
 490 }
 491
 492 /**
 493   Create IO submission queue.
 494
 495   @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 496
 497   @return EFI_SUCCESS    Successfully create io submission queue.
 498   @return others         Fail to create io submission queue.
 499
 500 **/
 501 EFI_STATUS
 502 NvmeCreateIoSubmissionQueue (
 503   IN PEI_NVME_CONTROLLER_PRIVATE_DATA    *Private
 504   )
 505 {
 506   EDKII_PEI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET    CommandPacket;
 507   EDKII_PEI_NVM_EXPRESS_COMMAND                     Command;
 508   EDKII_PEI_NVM_EXPRESS_COMPLETION                  Completion;
 509   EFI_STATUS                                        Status;
 510   NVME_ADMIN_CRIOSQ                                 CrIoSq;
 511
 512   ZeroMem (&CommandPacket, sizeof(EDKII_PEI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
 513   ZeroMem (&Command, sizeof(EDKII_PEI_NVM_EXPRESS_COMMAND));
 514   ZeroMem (&Completion, sizeof(EDKII_PEI_NVM_EXPRESS_COMPLETION));
 515   ZeroMem (&CrIoSq, sizeof(NVME_ADMIN_CRIOSQ));
 516
 517   CommandPacket.NvmeCmd        = &Command;
 518   CommandPacket.NvmeCompletion = &Completion;
 519
 520   Command.Cdw0.Opcode = NVME_ADMIN_CRIOSQ_CMD;
 521   Command.Cdw0.Cid    = Private->Cid[NVME_ADMIN_QUEUE]++;
 522   CommandPacket.TransferBuffer = Private->SqBuffer[NVME_IO_QUEUE];
 523   CommandPacket.TransferLength = EFI_PAGE_SIZE;
 524   CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
 525   CommandPacket.QueueType      = NVME_ADMIN_QUEUE;
 526
 527   CrIoSq.Qid   = NVME_IO_QUEUE;
 528   CrIoSq.Qsize = NVME_CSQ_SIZE;
 529   CrIoSq.Pc    = 1;
 530   CrIoSq.Cqid  = NVME_IO_QUEUE;
 531   CrIoSq.Qprio = 0;
 532   CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoSq, sizeof (NVME_ADMIN_CRIOSQ));
 533   CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
 534
 535   Status = NvmePassThru (
 536              Private,
 537              NVME_CONTROLLER_NSID,
 538              &CommandPacket
 539              );
 540   return Status;
 541 }
 542
 543 /**
 544   Initialize the Nvm Express controller.
 545
 546   @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 547
 548   @retval EFI_SUCCESS    The NVM Express Controller is initialized successfully.
 549   @retval Others         A device error occurred while initializing the controller.
 550
 551 **/
 552 EFI_STATUS
 553 NvmeControllerInit (
 554   IN PEI_NVME_CONTROLLER_PRIVATE_DATA    *Private
 555   )
 556 {
 557   EFI_STATUS    Status;
 558   UINTN         Index;
 559   NVME_AQA      Aqa;
 560   NVME_ASQ      Asq;
 561   NVME_ACQ      Acq;
 562   NVME_VER      Ver;
 563
 564   //
 565   // Dump the NVME controller implementation version
 566   //
 567   NVME_GET_VER (Private, &Ver);
 568   DEBUG ((DEBUG_INFO, "NVME controller implementation version: %d.%d\n", Ver.Mjr, Ver.Mnr));
 569
 570   //
 571   // Read the controller Capabilities register and verify that the NVM command set is supported
 572   //
 573   NVME_GET_CAP (Private, &Private->Cap);
 574   if (Private->Cap.Css != 0x01) {
 575     DEBUG ((DEBUG_ERROR, "%a: The NVME controller doesn't support NVMe command set.\n", __FUNCTION__));
 576     return EFI_UNSUPPORTED;
 577   }
 578
 579   //
 580   // Currently, the driver only supports 4k page size
 581   //
 582   if ((Private->Cap.Mpsmin + 12) > EFI_PAGE_SHIFT) {
 583     DEBUG ((DEBUG_ERROR, "%a: The driver doesn't support page size other than 4K.\n", __FUNCTION__));
 584     ASSERT (FALSE);
 585     return EFI_UNSUPPORTED;
 586   }
 587
 588   for (Index = 0; Index < NVME_MAX_QUEUES; Index++) {
 589     Private->Pt[Index]  = 0;
 590     Private->Cid[Index] = 0;
 591     ZeroMem ((VOID *)(UINTN)(&Private->SqTdbl[Index]), sizeof (NVME_SQTDBL));
 592     ZeroMem ((VOID *)(UINTN)(&Private->CqHdbl[Index]), sizeof (NVME_CQHDBL));
 593   }
 594   ZeroMem (Private->Buffer, EFI_PAGE_SIZE * NVME_MEM_MAX_PAGES);
 595
 596   //
 597   // Disable the NVME controller first
 598   //
 599   Status = NvmeDisableController (Private);
 600   if (EFI_ERROR (Status)) {
 601     DEBUG ((DEBUG_ERROR, "%a: NvmeDisableController fail, Status - %r\n", __FUNCTION__, Status));
 602     return Status;
 603   }
 604
 605   //
 606   // Set the number of entries in admin submission & completion queues
 607   //
 608   Aqa.Asqs  = NVME_ASQ_SIZE;
 609   Aqa.Rsvd1 = 0;
 610   Aqa.Acqs  = NVME_ACQ_SIZE;
 611   Aqa.Rsvd2 = 0;
 612
 613   //
 614   // Address of admin submission & completion queues
 615   //
 616   Asq = (UINT64)(UINTN)(NVME_ASQ_BASE (Private) & ~0xFFF);
 617   Acq = (UINT64)(UINTN)(NVME_ACQ_BASE (Private) & ~0xFFF);
 618
 619   //
 620   // Address of I/O submission & completion queues
 621   //
 622   Private->SqBuffer[0] = (NVME_SQ *)(UINTN)NVME_ASQ_BASE (Private);    // NVME_ADMIN_QUEUE
 623   Private->CqBuffer[0] = (NVME_CQ *)(UINTN)NVME_ACQ_BASE (Private);    // NVME_ADMIN_QUEUE
 624   Private->SqBuffer[1] = (NVME_SQ *)(UINTN)NVME_SQ_BASE (Private, 0);  // NVME_IO_QUEUE
 625   Private->CqBuffer[1] = (NVME_CQ *)(UINTN)NVME_CQ_BASE (Private, 0);  // NVME_IO_QUEUE
 626   DEBUG ((DEBUG_INFO, "Admin Submission Queue Size (Aqa.Asqs) = [%08X]\n", Aqa.Asqs));
 627   DEBUG ((DEBUG_INFO, "Admin Completion Queue Size (Aqa.Acqs) = [%08X]\n", Aqa.Acqs));
 628   DEBUG ((DEBUG_INFO, "Admin Submission Queue (SqBuffer[0]) =   [%08X]\n", Private->SqBuffer[0]));
 629   DEBUG ((DEBUG_INFO, "Admin Completion Queue (CqBuffer[0]) =   [%08X]\n", Private->CqBuffer[0]));
 630   DEBUG ((DEBUG_INFO, "I/O   Submission Queue (SqBuffer[1]) =   [%08X]\n", Private->SqBuffer[1]));
 631   DEBUG ((DEBUG_INFO, "I/O   Completion Queue (CqBuffer[1]) =   [%08X]\n", Private->CqBuffer[1]));
 632
 633   //
 634   // Program admin queue attributes
 635   //
 636   NVME_SET_AQA (Private, &Aqa);
 637
 638   //
 639   // Program admin submission & completion queues address
 640   //
 641   NVME_SET_ASQ (Private, &Asq);
 642   NVME_SET_ACQ (Private, &Acq);
 643
 644   //
 645   // Enable the NVME controller
 646   //
 647   Status = NvmeEnableController (Private);
 648   if (EFI_ERROR (Status)) {
 649     DEBUG ((DEBUG_ERROR, "%a: NvmeEnableController fail, Status - %r\n", __FUNCTION__, Status));
 650     return Status;
 651   }
 652
 653   //
 654   // Get the Identify Controller data
 655   //
 656   if (Private->ControllerData == NULL) {
 657     Private->ControllerData = (NVME_ADMIN_CONTROLLER_DATA *)AllocateZeroPool (sizeof (NVME_ADMIN_CONTROLLER_DATA));
 658     if (Private->ControllerData == NULL) {
 659       return EFI_OUT_OF_RESOURCES;
 660     }
 661   }
 662   Status = NvmeIdentifyController (Private, Private->ControllerData);
 663   if (EFI_ERROR (Status)) {
 664     DEBUG ((DEBUG_ERROR, "%a: NvmeIdentifyController fail, Status - %r\n", __FUNCTION__, Status));
 665     return Status;
 666   }
 667   NvmeDumpControllerData (Private->ControllerData);
 668
 669   //
 670   // Check the namespace number for storing the namespaces information
 671   //
 672   if (Private->ControllerData->Nn > MAX_UINT32 / sizeof (PEI_NVME_NAMESPACE_INFO)) {
 673     DEBUG ((
 674       DEBUG_ERROR,
 675       "%a: Number of Namespaces field in Identify Controller data not supported by the driver.\n",
 676       __FUNCTION__
 677       ));
 678     return EFI_UNSUPPORTED;
 679   }
 680
 681   //
 682   // Create one I/O completion queue and one I/O submission queue
 683   //
 684   Status = NvmeCreateIoCompletionQueue (Private);
 685   if (EFI_ERROR (Status)) {
 686     DEBUG ((DEBUG_ERROR, "%a: Create IO completion queue fail, Status - %r\n", __FUNCTION__, Status));
 687     return Status;
 688   }
 689   Status = NvmeCreateIoSubmissionQueue (Private);
 690   if (EFI_ERROR (Status)) {
 691     DEBUG ((DEBUG_ERROR, "%a: Create IO submission queue fail, Status - %r\n", __FUNCTION__, Status));
 692   }
 693
 694   return Status;
 695 }
 696
 697 /**
 698   Free the DMA resources allocated by an NVME controller.
 699
 700   @param[in] Private     The pointer to the PEI_NVME_CONTROLLER_PRIVATE_DATA data structure.
 701
 702 **/
 703 VOID
 704 NvmeFreeDmaResource (
 705   IN PEI_NVME_CONTROLLER_PRIVATE_DATA    *Private
 706   )
 707 {
 708   ASSERT (Private != NULL);
 709
 710   if (Private->BufferMapping != NULL) {
 711     IoMmuFreeBuffer (
 712        NVME_MEM_MAX_PAGES,
 713        Private->Buffer,
 714        Private->BufferMapping
 715        );
 716   }
 717
 718   return;
 719 }